Apply 12
Brady Martin
2025-05-01
Prompt 1: I provided ChatGPT with the following prompt and data: The
goal of this analysis is to build a predictive model that determines
whether a CEO was dismissed, using the CEO departures dataset. Please
clean the data cleaning, and tune an XGBoost classification model and
used H2O AutoML model for comparison in model accuracy. Additionally the
following is a glimpse of the dataset as well as a skim of the dataset.
data_clean %>% skimr::skim() ── Data Summary ────────────────────────
Values
Name Piped data Number of rows 7475
Number of columns 8
_______________________
Column type frequency:
factor 3
numeric 5
________________________
Group variables None
── Variable type: factor
───────────────────────────────────────────────────────────────────────────────
skim_variable n_missing complete_rate ordered n_unique top_counts
1 coname 0 1 FALSE 3427 BAR: 8, CLA: 8, FED: 8, GRE: 8 2 exec_fullname 0
1 FALSE 6975 Joh: 4, Mel: 4, Alb: 3, Ami: 3 3 ceo_dismissal 0 1 FALSE 2
not: 5992, dis: 1483
── Variable type: numeric ────────────────────────────────────────────────────────────────────────────── skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist 1 dismissal_dataset_id 0 1 5570. 25757. 1 2176. 4326 6580. 559044 ▇▁▁▁▁ 2 co_per_rol 0 1 21447. 16350. -1 6981 18269 33418. 64601 ▇▅▅▂▁ 3 tenure_no_ceodb 0 1 1.03 0.164 1 1 1 1 3 ▇▁▁▁▁ 4 max_tenure_ceodb 0 1 1.05 0.228 1 1 1 1 4 ▇▁▁▁▁ 5 fyear_gone 0 1 2006. 7.50 1980 2000 2006 2013 2021 ▁▂▇▇▆
data %>% glimpse() Rows: 9,423 Columns: 19 $ dismissal_dataset_id
_merge
Name Piped data Number of rows 9423
Number of columns 19
_______________________
Column type frequency:
character 8
numeric 10
POSIXct 1
________________________
Group variables None
── Variable type: character ─────────────────────────────────────────────────────────────────────────────────────────────── skim_variable n_missing complete_rate min max empty n_unique whitespace 1 coname 0 1 2 30 0 3860 0 2 exec_fullname 0 1 5 790 0 8701 0 3 interim_coceo 9105 0.0337 6 7 0 6 0 4 still_there 7311 0.224 3 10 0 77 0 5 notes 1644 0.826 5 3117 0 7755 0 6 sources 1475 0.843 18 1843 0 7915 0 7 eight_ks 4499 0.523 69 3884 0 4914 0 8 _merge 0 1 11 11 0 1 0
── Variable type: numeric ───────────────────────────────────────────────────────────────────────────────────────────────── skim_variable n_missing complete_rate mean sd p0 p25 p50 p75 p100 hist 1 dismissal_dataset_id 0 1 5684. 25005. 1 2306. 4593 6812. 559044 ▇▁▁▁▁ 2 gvkey 0 1 40132. 53921. 1004 7337 14385 60900. 328795 ▇▁▁▁▁ 3 fyear 0 1 2008. 8.19 1987 2000 2008 2016 2020 ▁▆▅▅▇ 4 co_per_rol 0 1 25580. 18202. -1 8556. 22980 39276. 64602 ▇▆▅▃▃ 5 departure_code 1667 0.823 5.20 1.53 1 5 5 7 9 ▁▃▇▅▁ 6 ceo_dismissal 1813 0.808 0.196 0.397 0 0 0 0 1 ▇▁▁▁▂ 7 tenure_no_ceodb 0 1 1.03 0.167 0 1 1 1 3 ▁▇▁▁▁ 8 max_tenure_ceodb 0 1 1.05 0.235 1 1 1 1 4 ▇▁▁▁▁ 9 fyear_gone 1802 0.809 2007. 13.6 1980 2000 2007 2013 2997 ▇▁▁▁▁ 10 cik 245 0.974 741469. 486551. 1750 106413 857323 1050376. 1808065 ▆▁▇▂▁
── Variable type: POSIXct ───────────────────────────────────────────────────────────────────────────────────────────────── skim_variable n_missing complete_rate min max median n_unique 1 leftofc 1802 0.809 1981-01-01 00:00:00 2998-04-27 00:00:00 2006-12-31 00:00:00 3627
data <- read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/main/data/2021/2021-04-27/departures.csv')## Rows: 9423 Columns: 19
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (8): coname, exec_fullname, interim_coceo, still_there, notes, sources...
## dbl (10): dismissal_dataset_id, gvkey, fyear, co_per_rol, departure_code, c...
## dttm (1): leftofc
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.data_clean <- data %>%
filter(!is.na(ceo_dismissal)) %>%
mutate(
ceo_dismissal = if_else(ceo_dismissal == 1, "dismissed", "not_dis") %>% as.factor()
) %>%
select(-c(
still_there, interim_coceo, eight_ks, notes,
`_merge`, sources, departure_code,
cik, gvkey, fyear, leftofc
)) %>%
distinct(dismissal_dataset_id, .keep_all = TRUE) %>%
filter(fyear_gone < 2025) %>%
mutate(across(where(is.character), as.factor)) %>%
na.omit()
data_clean %>% skimr::skim()| Name | Piped data |
| Number of rows | 7475 |
| Number of columns | 8 |
| _______________________ | |
| Column type frequency: | |
| factor | 3 |
| numeric | 5 |
| ________________________ | |
| Group variables | None |
Variable type: factor
| skim_variable | n_missing | complete_rate | ordered | n_unique | top_counts |
|---|---|---|---|---|---|
| coname | 0 | 1 | FALSE | 3427 | BAR: 8, CLA: 8, FED: 8, GRE: 8 |
| exec_fullname | 0 | 1 | FALSE | 6975 | Joh: 4, Mel: 4, Alb: 3, Ami: 3 |
| ceo_dismissal | 0 | 1 | FALSE | 2 | not: 5992, dis: 1483 |
Variable type: numeric
| skim_variable | n_missing | complete_rate | mean | sd | p0 | p25 | p50 | p75 | p100 | hist |
|---|---|---|---|---|---|---|---|---|---|---|
| dismissal_dataset_id | 0 | 1 | 5570.32 | 25757.33 | 1 | 2175.5 | 4326 | 6579.5 | 559044 | ▇▁▁▁▁ |
| co_per_rol | 0 | 1 | 21446.53 | 16350.34 | -1 | 6981.0 | 18269 | 33418.5 | 64601 | ▇▅▅▂▁ |
| tenure_no_ceodb | 0 | 1 | 1.03 | 0.16 | 1 | 1.0 | 1 | 1.0 | 3 | ▇▁▁▁▁ |
| max_tenure_ceodb | 0 | 1 | 1.05 | 0.23 | 1 | 1.0 | 1 | 1.0 | 4 | ▇▁▁▁▁ |
| fyear_gone | 0 | 1 | 2006.41 | 7.50 | 1980 | 2000.0 | 2006 | 2013.0 | 2021 | ▁▂▇▇▆ |
set.seed(2025)
data_split <- initial_split(data_clean, strata = ceo_dismissal)
data_train <- training(data_split)
data_test <- testing(data_split)
data_cv <- vfold_cv(data_train, v = 5, strata = ceo_dismissal)xgboost_rec <- recipe(ceo_dismissal ~ ., data = data_train) %>%
update_role(dismissal_dataset_id, new_role = "ID") %>%
step_other(exec_fullname, coname, threshold = 0.01) %>%
step_dummy(all_nominal_predictors()) %>%
step_YeoJohnson(max_tenure_ceodb) %>%
step_normalize(all_numeric_predictors()) %>%
step_smote(ceo_dismissal)xgboost_spec <- boost_tree(
trees = tune(),
tree_depth = tune(),
min_n = tune(),
mtry = tune(),
learn_rate = tune()
) %>%
set_mode("classification") %>%
set_engine("xgboost")
xgboost_workflow <- workflow() %>%
add_recipe(xgboost_rec) %>%
add_model(xgboost_spec)Prompt 2:
“I got this error how do I fix it ‘Error in check_grid(): ! The provided grid is missing the following parameter columns that have been marked for tuning by tune(): ’mtry’. Backtrace: ▆ 1. ├─tune::tune_grid(…) 2. └─tune:::tune_grid.workflow(…) 3. └─tune:::tune_grid_workflow(…) 4. └─tune:::check_grid(grid = grid, workflow = workflow, pset = pset) 5. └─rlang::abort(msg)
Quitting from lines 85-105 [unnamed-chunk-5] (Apply_12.Rmd) Execution halted’”
I was provided with the response “Update this block:
tree_grid <- grid_regular( trees(), tree_depth(), min_n(), learn_rate(), levels = 3 )
To this:
tree_grid <- grid_regular(trees(), tree_depth(), levels = 5) )
tree_grid <- grid_regular(trees(),
tree_depth(),
levels = 5)
doParallel::registerDoParallel()
set.seed(17375)
xgboost_tune <-
tune_grid(xgboost_workflow,
resamples = data_cv,
grid = 5,
control = control_grid(save_pred = TRUE))## i Creating pre-processing data to finalize unknown parameter: mtrycollect_metrics(xgboost_tune)## # A tibble: 15 × 11
## mtry trees min_n tree_depth learn_rate .metric .estimator mean n
## <int> <int> <int> <int> <dbl> <chr> <chr> <dbl> <int>
## 1 1 1674 21 8 0.0166 accuracy binary 0.536 5
## 2 1 1674 21 8 0.0166 brier_class binary 0.240 5
## 3 1 1674 21 8 0.0166 roc_auc binary 0.584 5
## 4 2 346 39 12 0.128 accuracy binary 0.575 5
## 5 2 346 39 12 0.128 brier_class binary 0.237 5
## 6 2 346 39 12 0.128 roc_auc binary 0.577 5
## 7 4 852 17 2 0.00405 accuracy binary 0.537 5
## 8 4 852 17 2 0.00405 brier_class binary 0.243 5
## 9 4 852 17 2 0.00405 roc_auc binary 0.584 5
## 10 5 1497 27 6 0.0475 accuracy binary 0.610 5
## 11 5 1497 27 6 0.0475 brier_class binary 0.230 5
## 12 5 1497 27 6 0.0475 roc_auc binary 0.578 5
## 13 6 659 8 14 0.00288 accuracy binary 0.580 5
## 14 6 659 8 14 0.00288 brier_class binary 0.235 5
## 15 6 659 8 14 0.00288 roc_auc binary 0.576 5
## # ℹ 2 more variables: std_err <dbl>, .config <chr>collect_predictions(xgboost_tune) %>%
group_by(id) %>%
roc_curve(ceo_dismissal, .pred_dismissed) %>%
autoplot()
Prompt 3:
“This is the error I got how do I fix it? ’Error in select_best(): ! … must be empty. ✖ Problematic argument: • ..1 =”accuracy” ℹ Did you forget to name an argument? Backtrace: ▆ 1. ├─… %>% last_fit(data_split) 2. ├─tune::last_fit(., data_split) 3. ├─tune::finalize_workflow(., select_best(xgboost_tune, “accuracy”)) 4. │ └─tune:::check_final_param(parameters) 5. ├─tune::select_best(xgboost_tune, “accuracy”) 6. └─tune:::select_best.tune_results(xgboost_tune, “accuracy”) 7. └─rlang::check_dots_empty() 8. └─rlang:::action_dots(…) 9. ├─base (local) try_dots(…) 10. └─rlang (local) action(…)
Quitting from lines 109-118 [unnamed-chunk-7] (Apply_12.Rmd) Execution halted’” I was provided with the response “Change this line:
select_best(xgboost_tune, “accuracy”)
To:
select_best(xgboost_tune, metric = “accuracy”)”
xgboost_final <- xgboost_workflow %>%
finalize_workflow(select_best(xgboost_tune, metric = "accuracy")) %>%
last_fit(data_split)
collect_metrics(xgboost_final)## # A tibble: 3 × 4
## .metric .estimator .estimate .config
## <chr> <chr> <dbl> <chr>
## 1 accuracy binary 0.589 Preprocessor1_Model1
## 2 roc_auc binary 0.561 Preprocessor1_Model1
## 3 brier_class binary 0.235 Preprocessor1_Model1collect_predictions(xgboost_final) %>%
conf_mat(ceo_dismissal, .pred_class) %>%
autoplot()
xgboost_final %>%
extract_fit_engine() %>%
vip()
recipe_obj <- recipe(ceo_dismissal ~ ., data = data_train) %>%
step_zv(all_predictors())h2o.init()## Connection successful!
##
## R is connected to the H2O cluster:
## H2O cluster uptime: 3 days 12 hours
## H2O cluster timezone: America/New_York
## H2O data parsing timezone: UTC
## H2O cluster version: 3.44.0.3
## H2O cluster version age: 1 year, 4 months and 15 days
## H2O cluster name: H2O_started_from_R_bradymartin_hek320
## H2O cluster total nodes: 1
## H2O cluster total memory: 3.04 GB
## H2O cluster total cores: 8
## H2O cluster allowed cores: 8
## H2O cluster healthy: TRUE
## H2O Connection ip: localhost
## H2O Connection port: 54321
## H2O Connection proxy: NA
## H2O Internal Security: FALSE
## R Version: R version 4.4.2 (2024-10-31)## Warning in h2o.clusterInfo():
## Your H2O cluster version is (1 year, 4 months and 15 days) old. There may be a newer version available.
## Please download and install the latest version from: https://h2o-release.s3.amazonaws.com/h2o/latest_stable.htmlsplit.h2o <- h2o.splitFrame(as.h2o(data_train), ratios = 0.85, seed = 2025)## | | | 0% | |======================================================================| 100%train_h2o <- split.h2o[[1]]
valid_h2o <- split.h2o[[2]]
test_h2o <- as.h2o(data_test)## | | | 0% | |======================================================================| 100%y <- "ceo_dismissal"
x <- setdiff(names(data_train), c(y))
models_h2o <- h2o.automl(
x = x,
y = y,
training_frame = train_h2o,
validation_frame = valid_h2o,
leaderboard_frame = test_h2o,
max_models = 10,
exclude_algos = "DeepLearning",
nfolds = 5,
seed = 2025
)## | | | 0% | |=== | 4%
## 11:54:58.255: User specified a validation frame with cross-validation still enabled. Please note that the models will still be validated using cross-validation only, the validation frame will be used to provide purely informative validation metrics on the trained models.
## 11:54:58.257: AutoML: XGBoost is not available; skipping it. | |=============== | 21% | |==================== | 29% | |=============================================== | 67% | |======================================================================| 100%leader_model <- h2o.get_best_model(models_h2o)
performance_h2o <- h2o.performance(leader_model, newdata = test_h2o)
h2o.auc(performance_h2o)## [1] 0.5867554h2o.accuracy(performance_h2o)## threshold accuracy
## 1 0.9421074 0.1990369
## 2 0.9389176 0.2001070
## 3 0.9369205 0.2017121
## 4 0.9353372 0.2027822
## 5 0.9341904 0.2033173
##
## ---
## threshold accuracy
## 395 0.4506053 0.7998930
## 396 0.4361137 0.8004280
## 397 0.4313271 0.7998930
## 398 0.4052392 0.8004280
## 399 0.3792865 0.8009631
## 400 0.3446967 0.8014981h2o.confusionMatrix(performance_h2o)## Confusion Matrix (vertical: actual; across: predicted) for max f1 @ threshold = 0.344696694305184:
## dismissed not_dis Error Rate
## dismissed 0 371 1.000000 =371/371
## not_dis 0 1498 0.000000 =0/1498
## Totals 0 1869 0.198502 =371/1869predictions <- h2o.predict(leader_model, test_h2o) %>% as_tibble()## | | | 0% | |======================================================================| 100%## Warning in doTryCatch(return(expr), name, parentenv, handler): Test/Validation
## dataset column 'coname' has levels not trained on: ["3D SYSTEMS CORP", "AAON
## INC", "ACADIA HEALTHCARE CO INC", "ACCESS HEALTH INC", "ACTEL CORP", "ACUITY
## BRANDS INC", "ACUITY CIMATRIX INC", "ADESA INC", "ADIENT PLC", "ADVANCED TISSUE
## SCI -CL A", ...360 not listed..., "WESTWOOD ONE INC -OLD", "WEX INC", "WGL
## HOLDINGS INC", "WILSHIRE BANCORP INC", "WORLD FUEL SERVICES CORP", "WYLE
## ELECTRONICS", "WYNN'S INTERNATIONAL INC", "XIRCOM INC", "ZEP INC", "ZERO
## CORP/DE"]## Warning in doTryCatch(return(expr), name, parentenv, handler): Test/Validation
## dataset column 'exec_fullname' has levels not trained on: ["A. E. Benton", "A.
## Earl Swift", "A. Eugene Sapp, Jr.", "A. James Dearlove", "A. Lorne Weil",
## "Aaron D. Todd III", "Aaron William Regent F.C.A. FCPA B.A. C.A", "Abhijit Y.
## Talwalkar", "Abraham N. Reichental", "Adam D. Singer", ...1624 not listed...,
## "William V. Murray", "William W. Boyle MBA", "William W. Lovette", "William
## White Adams", "William Wrigley Jr.", "Willis J. Johnson", "Wilson W. Cheung",
## "Yaron I. Eitan", "Ying Lu", "Zan Guerry"]predictions %>%
bind_cols(data_test) %>%
select(ceo_dismissal, predict, everything())## # A tibble: 1,869 × 11
## ceo_dismissal predict dismissed not_dis dismissal_dataset_id coname
## <fct> <fct> <dbl> <dbl> <dbl> <fct>
## 1 not_dis not_dis 0.125 0.875 76 ALBERTO-CULVER …
## 2 not_dis not_dis 0.125 0.875 81 ALCAN INC
## 3 not_dis not_dis 0.108 0.892 109 ALCOA INC
## 4 not_dis not_dis 0.121 0.879 117 HESS CORP
## 5 dismissed not_dis 0.131 0.869 119 AMDAHL CORP
## 6 not_dis not_dis 0.0934 0.907 122 BEAM INC
## 7 not_dis not_dis 0.132 0.868 138 AMERICAN ELECTR…
## 8 not_dis not_dis 0.132 0.868 139 AMERICAN ELECTR…
## 9 dismissed not_dis 0.132 0.868 143 AMERICAN EXPRES…
## 10 not_dis not_dis 0.130 0.870 150 AMERICAN GENERA…
## # ℹ 1,859 more rows
## # ℹ 5 more variables: co_per_rol <dbl>, exec_fullname <fct>,
## # tenure_no_ceodb <dbl>, max_tenure_ceodb <dbl>, fyear_gone <dbl>