Goal is to automate building and tuning a classification model to predict sellable online, using the h2o::h2o.automl.

Set up

Import data

Import the cleaned data from Module 7.

library(h2o)
## Warning: package 'h2o' was built under R version 4.5.3
## 
## ----------------------------------------------------------------------
## 
## Your next step is to start H2O:
##     > h2o.init()
## 
## For H2O package documentation, ask for help:
##     > ??h2o
## 
## After starting H2O, you can use the Web UI at http://localhost:54321
## For more information visit https://docs.h2o.ai
## 
## ----------------------------------------------------------------------
## 
## Attaching package: 'h2o'
## The following objects are masked from 'package:stats':
## 
##     cor, sd, var
## The following objects are masked from 'package:base':
## 
##     %*%, %in%, &&, ||, apply, as.factor, as.numeric, colnames,
##     colnames<-, ifelse, is.character, is.factor, is.numeric, log,
##     log10, log1p, log2, round, signif, trunc
library(tidyverse)
## ── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
## ✔ dplyr     1.1.4     ✔ readr     2.1.5
## ✔ forcats   1.0.0     ✔ stringr   1.5.2
## ✔ ggplot2   4.0.0     ✔ tibble    3.3.0
## ✔ lubridate 1.9.4     ✔ tidyr     1.3.1
## ✔ purrr     1.1.0
## ── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ lubridate::day()   masks h2o::day()
## ✖ dplyr::filter()    masks stats::filter()
## ✖ lubridate::hour()  masks h2o::hour()
## ✖ dplyr::lag()       masks stats::lag()
## ✖ lubridate::month() masks h2o::month()
## ✖ lubridate::week()  masks h2o::week()
## ✖ lubridate::year()  masks h2o::year()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
library(tidymodels)
## Warning: package 'tidymodels' was built under R version 4.5.2
## ── Attaching packages ────────────────────────────────────── tidymodels 1.4.1 ──
## ✔ broom        1.0.12     ✔ rsample      1.3.1 
## ✔ dials        1.4.2      ✔ tailor       0.1.0 
## ✔ infer        1.1.0      ✔ tune         2.0.1 
## ✔ modeldata    1.5.1      ✔ workflows    1.3.0 
## ✔ parsnip      1.4.1      ✔ workflowsets 1.1.1 
## ✔ recipes      1.3.1      ✔ yardstick    1.3.2
## Warning: package 'broom' was built under R version 4.5.3
## Warning: package 'dials' was built under R version 4.5.2
## Warning: package 'infer' was built under R version 4.5.2
## Warning: package 'modeldata' was built under R version 4.5.2
## Warning: package 'parsnip' was built under R version 4.5.2
## Warning: package 'tailor' was built under R version 4.5.2
## Warning: package 'tune' was built under R version 4.5.2
## Warning: package 'workflows' was built under R version 4.5.2
## Warning: package 'workflowsets' was built under R version 4.5.2
## Warning: package 'yardstick' was built under R version 4.5.2
## ── Conflicts ───────────────────────────────────────── tidymodels_conflicts() ──
## ✖ scales::discard() masks purrr::discard()
## ✖ dplyr::filter()   masks stats::filter()
## ✖ recipes::fixed()  masks stringr::fixed()
## ✖ dplyr::lag()      masks stats::lag()
## ✖ yardstick::spec() masks readr::spec()
## ✖ recipes::step()   masks stats::step()
library(tidyquant)
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo 
## ── Attaching core tidyquant packages ─────────────────────── tidyquant 1.0.11 ──
## ✔ PerformanceAnalytics 2.0.8      ✔ TTR                  0.24.4
## ✔ quantmod             0.4.28     ✔ xts                  0.14.1── Conflicts ────────────────────────────────────────── tidyquant_conflicts() ──
## ✖ zoo::as.Date()                 masks base::as.Date()
## ✖ zoo::as.Date.numeric()         masks base::as.Date.numeric()
## ✖ scales::col_factor()           masks readr::col_factor()
## ✖ lubridate::day()               masks h2o::day()
## ✖ scales::discard()              masks purrr::discard()
## ✖ dplyr::filter()                masks stats::filter()
## ✖ xts::first()                   masks dplyr::first()
## ✖ recipes::fixed()               masks stringr::fixed()
## ✖ lubridate::hour()              masks h2o::hour()
## ✖ dplyr::lag()                   masks stats::lag()
## ✖ xts::last()                    masks dplyr::last()
## ✖ PerformanceAnalytics::legend() masks graphics::legend()
## ✖ TTR::momentum()                masks dials::momentum()
## ✖ lubridate::month()             masks h2o::month()
## ✖ yardstick::spec()              masks readr::spec()
## ✖ quantmod::summary()            masks h2o::summary(), base::summary()
## ✖ lubridate::week()              masks h2o::week()
## ✖ lubridate::year()              masks h2o::year()
## ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors
data <- read_csv("../00_data/data_wrangled/data_clean_ikea.csv") %>%
    
    # h2o requires all variables to be either numeric or factors
    mutate(across(where(is.character), factor))
## Rows: 1899 Columns: 11
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (5): name, category, other_colors, short_description, designer
## dbl (5): item_id, price, depth, height, width
## lgl (1): sellable_online
## 
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

Split data

set.seed(1234)

data_split <- initial_split(data, strata = "sellable_online")
train_tbl <- training(data_split)
test_tbl <- testing(data_split)

Recipes

recipe_obj <- recipe(sellable_online ~ ., data = train_tbl) %>%
    
    # Remove zero variance variables
    step_zv(all_predictors()) 

Model

# Initialize h2o
h2o.init()
## 
## H2O is not running yet, starting it now...
## 
## Note:  In case of errors look at the following log files:
##     C:\Users\liamc\AppData\Local\Temp\RtmpcV1KAw\file787845d23253/h2o_liamc_started_from_r.out
##     C:\Users\liamc\AppData\Local\Temp\RtmpcV1KAw\file78781ee96449/h2o_liamc_started_from_r.err
## 
## 
## Starting H2O JVM and connecting:  Connection successful!
## 
## R is connected to the H2O cluster: 
##     H2O cluster uptime:         2 seconds 367 milliseconds 
##     H2O cluster timezone:       America/New_York 
##     H2O data parsing timezone:  UTC 
##     H2O cluster version:        3.44.0.3 
##     H2O cluster version age:    2 years, 4 months and 3 days 
##     H2O cluster name:           H2O_started_from_R_liamc_kmo444 
##     H2O cluster total nodes:    1 
##     H2O cluster total memory:   7.94 GB 
##     H2O cluster total cores:    24 
##     H2O cluster allowed cores:  24 
##     H2O cluster healthy:        TRUE 
##     H2O Connection ip:          localhost 
##     H2O Connection port:        54321 
##     H2O Connection proxy:       NA 
##     H2O Internal Security:      FALSE 
##     R Version:                  R version 4.5.1 (2025-06-13 ucrt)
## Warning in h2o.clusterInfo(): 
## Your H2O cluster version is (2 years, 4 months and 3 days) old. There may be a newer version available.
## Please download and install the latest version from: https://h2o-release.s3.amazonaws.com/h2o/latest_stable.html
split.h2o <- h2o.splitFrame(as.h2o(train_tbl), ratios = c(0.85), seed = 2345)
##   |                                                                              |                                                                      |   0%  |                                                                              |======================================================================| 100%
train_h2o <- split.h2o[[1]]
valid_h2o <- split.h2o[[2]]
test_h2o  <- as.h2o(test_tbl)
##   |                                                                              |                                                                      |   0%  |                                                                              |======================================================================| 100%
y <- "sellable_online"
x <- setdiff(names(train_tbl), y)

models_h2o <- h2o.automl(
    x = x,
    y = y, 
    training_frame    = train_h2o,
    validation_frame  = valid_h2o, 
    leaderboard_frame = test_h2o, 
    # max_runtime_secs  = 30, 
    max_models        = 10, 
    exclude_algos     = "DeepLearning",
    nfolds            = 5, 
    seed              = 3456   
)
##   |                                                                              |                                                                      |   0%  |                                                                              |=                                                                     |   1%
## 17:29:25.105: User specified a validation frame with cross-validation still enabled. Please note that the models will still be validated using cross-validation only, the validation frame will be used to provide purely informative validation metrics on the trained models.
## 17:29:25.111: AutoML: XGBoost is not available; skipping it.  |                                                                              |======                                                                |   8%
## 
## Job $03017f00000132d4ffffffff$_8291a67a38291be44105749c90d64eed was cancelled.

Examine the output of h2o.automl

models_h2o %>% typeof()
## [1] "S4"
models_h2o %>% slotNames()
## [1] "project_name"   "leader"         "leaderboard"    "event_log"     
## [5] "modeling_steps" "training_info"
models_h2o@leaderboard
##                         model_id       auc    logloss     aucpr
## 1 GBM_1_AutoML_1_20260423_172925 0.9706921 0.04081237 0.9998106
## 2 GLM_1_AutoML_1_20260423_172925 0.9512712 0.02525709 0.9996682
## 3 DRF_1_AutoML_1_20260423_172925 0.8442797 0.03008068 0.9983703
## 4 GBM_3_AutoML_1_20260423_172925 0.7680085 0.04718649 0.9975400
## 5 GBM_2_AutoML_1_20260423_172925 0.7545904 0.06213847 0.9972256
##   mean_per_class_error       rmse         mse
## 1            0.5000000 0.07926326 0.006282665
## 2            0.3333333 0.06889723 0.004746829
## 3            0.5000000 0.07571374 0.005732570
## 4            0.5000000 0.07950201 0.006320569
## 5            0.5000000 0.07946625 0.006314885
## 
## [5 rows x 7 columns]
models_h2o@leader
## Model Details:
## ==============
## 
## H2OBinomialModel: gbm
## Model ID:  GBM_1_AutoML_1_20260423_172925 
## Model Summary: 
##   number_of_trees number_of_internal_trees model_size_in_bytes min_depth
## 1              49                       49               27443         1
##   max_depth mean_depth min_leaves max_leaves mean_leaves
## 1         8    6.14286          2          9     7.71429
## 
## 
## H2OBinomialMetrics: gbm
## ** Reported on training data. **
## 
## MSE:  0.0007091025
## RMSE:  0.02662898
## LogLoss:  0.002835019
## Mean Per-Class Error:  0.05555556
## AUC:  0.9999075
## AUCPR:  0.9999993
## Gini:  0.999815
## R^2:  0.9039507
## 
## Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
##        FALSE TRUE    Error     Rate
## FALSE      8    1 0.111111     =1/9
## TRUE       0 1201 0.000000  =0/1201
## Totals     8 1202 0.000826  =1/1210
## 
## Maximum Metrics: Maximum metrics at their respective thresholds
##                         metric threshold       value idx
## 1                       max f1  0.454676    0.999584 395
## 2                       max f2  0.454676    0.999833 395
## 3                 max f0point5  0.632906    0.999833 393
## 4                 max accuracy  0.632906    0.999174 393
## 5                max precision  0.999915    1.000000   0
## 6                   max recall  0.454676    1.000000 395
## 7              max specificity  0.999915    1.000000   0
## 8             max absolute_mcc  0.632906    0.948288 393
## 9   max min_per_class_accuracy  0.632906    0.999167 393
## 10 max mean_per_class_accuracy  0.632906    0.999584 393
## 11                     max tns  0.999915    9.000000   0
## 12                     max fns  0.999915 1200.000000   0
## 13                     max fps  0.071193    9.000000 399
## 14                     max tps  0.454676 1201.000000 395
## 15                     max tnr  0.999915    1.000000   0
## 16                     max fnr  0.999915    0.999167   0
## 17                     max fpr  0.071193    1.000000 399
## 18                     max tpr  0.454676    1.000000 395
## 
## Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`
## H2OBinomialMetrics: gbm
## ** Reported on validation data. **
## ** Validation metrics **
## 
## MSE:  0.004648113
## RMSE:  0.06817707
## LogLoss:  0.02749327
## Mean Per-Class Error:  0.5
## AUC:  0.9389671
## AUCPR:  0.9997058
## Gini:  0.8779343
## R^2:  0.0006338792
## 
## Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
##        FALSE TRUE    Error    Rate
## FALSE      0    1 1.000000    =1/1
## TRUE       0  213 0.000000  =0/213
## Totals     0  214 0.004673  =1/214
## 
## Maximum Metrics: Maximum metrics at their respective thresholds
##                         metric threshold      value idx
## 1                       max f1  0.973657   0.997658 207
## 2                       max f2  0.973657   0.999062 207
## 3                 max f0point5  0.973657   0.996258 207
## 4                 max accuracy  0.973657   0.995327 207
## 5                max precision  0.999947   1.000000   0
## 6                   max recall  0.973657   1.000000 207
## 7              max specificity  0.999947   1.000000   0
## 8             max absolute_mcc  0.997472   0.258977 194
## 9   max min_per_class_accuracy  0.997472   0.938967 194
## 10 max mean_per_class_accuracy  0.997472   0.969484 194
## 11                     max tns  0.999947   1.000000   0
## 12                     max fns  0.999947 212.000000   0
## 13                     max fps  0.996891   1.000000 195
## 14                     max tps  0.973657 213.000000 207
## 15                     max tnr  0.999947   1.000000   0
## 16                     max fnr  0.999947   0.995305   0
## 17                     max fpr  0.996891   1.000000 195
## 18                     max tpr  0.973657   1.000000 207
## 
## Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`
## H2OBinomialMetrics: gbm
## ** Reported on cross-validation data. **
## ** 5-fold cross-validation on training data (Metrics computed for combined holdout predictions) **
## 
## MSE:  0.003569446
## RMSE:  0.05974484
## LogLoss:  0.01851759
## Mean Per-Class Error:  0.2777778
## AUC:  0.9517994
## AUCPR:  0.9995493
## Gini:  0.9035989
## R^2:  0.5165116
## 
## Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
##        FALSE TRUE    Error     Rate
## FALSE      4    5 0.555556     =5/9
## TRUE       0 1201 0.000000  =0/1201
## Totals     4 1206 0.004132  =5/1210
## 
## Maximum Metrics: Maximum metrics at their respective thresholds
##                         metric threshold       value idx
## 1                       max f1  0.207468    0.997923 396
## 2                       max f2  0.207468    0.999168 396
## 3                 max f0point5  0.930103    0.997670 392
## 4                 max accuracy  0.930103    0.995868 392
## 5                max precision  1.000000    1.000000   0
## 6                   max recall  0.207468    1.000000 396
## 7              max specificity  1.000000    1.000000   0
## 8             max absolute_mcc  0.930103    0.705047 392
## 9   max min_per_class_accuracy  0.998733    0.888889 300
## 10 max mean_per_class_accuracy  0.994080    0.935285 373
## 11                     max tns  1.000000    9.000000   0
## 12                     max fns  1.000000 1172.000000   0
## 13                     max fps  0.002469    9.000000 399
## 14                     max tps  0.207468 1201.000000 396
## 15                     max tnr  1.000000    1.000000   0
## 16                     max fnr  1.000000    0.975853   0
## 17                     max fpr  0.002469    1.000000 399
## 18                     max tpr  0.207468    1.000000 396
## 
## Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`
## Cross-Validation Metrics Summary: 
##                             mean       sd cv_1_valid cv_2_valid cv_3_valid
## accuracy                0.997521 0.003696   1.000000   0.991735   1.000000
## auc                     0.924439 0.165090   1.000000   0.993027   1.000000
## err                     0.002479 0.003696   0.000000   0.008264   0.000000
## err_count               0.600000 0.894427   0.000000   2.000000   0.000000
## f0point5                0.998006 0.002974   1.000000   0.993350   1.000000
## f1                      0.998751 0.001863   1.000000   0.995833   1.000000
## f2                      0.999499 0.000747   1.000000   0.998329   1.000000
## lift_top_group          1.007510 0.004579   1.000000   1.012552   1.008333
## logloss                 0.020258 0.020610   0.007666   0.048666   0.000420
## max_per_class_error     0.233333 0.324893   0.000000   0.666667   0.000000
## mcc                     0.820147 0.214420         NA   0.574950   1.000000
## mean_per_class_accuracy 0.883333 0.162447   1.000000   0.666667   1.000000
## mean_per_class_error    0.116667 0.162447   0.000000   0.333333   0.000000
## mse                     0.003695 0.002993   0.002749   0.008272   0.000035
## pr_auc                  0.998865 0.002491   1.000000   0.999913   1.000000
## precision               0.997510 0.003711   1.000000   0.991701   1.000000
## r2                          -Inf       NA       -Inf   0.324390   0.995784
## recall                  1.000000 0.000000   1.000000   1.000000   1.000000
## rmse                    0.054155 0.030878   0.052432   0.090948   0.005879
## specificity             0.708333 0.343592         NA   0.333333   1.000000
##                         cv_4_valid cv_5_valid
## accuracy                  0.995868   1.000000
## auc                       0.629167   1.000000
## err                       0.004132   0.000000
## err_count                 1.000000   0.000000
## f0point5                  0.996678   1.000000
## f1                        0.997921   1.000000
## f2                        0.999167   1.000000
## lift_top_group            1.008333   1.008333
## logloss                   0.035152   0.009386
## max_per_class_error       0.500000   0.000000
## mcc                       0.705638   1.000000
## mean_per_class_accuracy   0.750000   1.000000
## mean_per_class_error      0.250000   0.000000
## mse                       0.004250   0.003172
## pr_auc                    0.994410   1.000000
## precision                 0.995851   1.000000
## r2                        0.481410   0.613012
## recall                    1.000000   1.000000
## rmse                      0.065195   0.056319
## specificity               0.500000   1.000000

Save and Load

#h2o.getModel
#h2o.saveModel
#h2o.loadModel

#h2o.getModel("GBM_4_AutoML_1_20260423_105606") %>%
#   h2o.saveModel("h2o_models/GBM_4_AutoML_1_20260423_105606")

best_model <- h2o.loadModel("h2o_models/GBM_4_AutoML_1_20260423_105606")

Make predictions

predictions <- h2o.predict(best_model, newdata = test_h2o)
##   |                                                                              |                                                                      |   0%  |                                                                              |======================================================================| 100%
## Warning in doTryCatch(return(expr), name, parentenv, handler): Test/Validation
## dataset column 'name' has levels not trained on: ["ALGOT / BROR", "EKENÄS",
## "ENETRI", "FALHOLMEN", "FEJAN", "GAMLEBY", "GENEVAD", "GERSBY", "GRUNDTAL",
## "GRÖNADAL", ...13 not listed..., "PAX / MEHAMN/AULI", "PAX / MEHAMN/SEKKEN",
## "PAX / VINGROM", "PAX / VINTERBRO", "RÅDVIKEN", "SKARPÖ", "SKRUVSTA",
## "STOCKHOLM 2017", "SVENARNE", "TÄRNÖ"]
## Warning in doTryCatch(return(expr), name, parentenv, handler): Test/Validation
## dataset column 'short_description' has levels not trained on: ["1 section,
## 84x40x216 cm", "1 section/shelves/drawers, 48x30x179 cm", "2 sections,
## 165x55x126 cm", "2 sections/shelves, 174x50x124 cm", "2-seat modular sofa,
## outdoor, 161x82x84 cm", "3 sections/shelves, 219x30x226 cm", "3
## sections/shelves, 259x30x226 cm", "3 sections/shelves, 259x50x226 cm", "3
## sections/shelves/cabinet, 259x30x124 cm", "3-seat modular sofa", ...134 not
## listed..., "Wardrobe combination, 250x60x236 cm", "Wardrobe with 2 doors+3
## drawers, 160x42x181 cm", "Wardrobe with 3 doors, 117x176 cm", "Wardrobe with 7
## doors+3 drawers, 240x57x221 cm", "Wardrobe, 175-200x57x251 cm", "Wardrobe,
## 175x58x201 cm", "Wardrobe, 240x57x123 cm", "Wardrobe, 240x57x251 cm", "Window
## table, 80x40x75 cm", "Wing chair"]
## Warning in doTryCatch(return(expr), name, parentenv, handler): Test/Validation
## dataset column 'designer' has levels not trained on: ["104.246.21 KNOPPARP sofa
## is very durable thanks to the metal construction and strong supporting
## fabric.Thanks to the innovative construction, we can use less materials and
## foam when we make KNOPPARP sofa, while the padded cover ensures that the
## comfort is maintained.A sofa with small, neat dimensions which is easy to
## furnish with, even when space is limited. This cover is made from KNISA fabric
## in polyester, which is dope-dyed. It’s a durable material which has a soft
## feel.The dope-dyeing process reduces consumption of water and dyestuff compared
## to traditional dyeing techniques.The cover is easy to keep clean as it is
## removable and can be machine washed.Easy to bring home if you choose to carry
## it on your own. The packaging is just over one metre in height and weighs 17
## kg.10 year guarantee. Read about the terms in the guarantee brochure.This
## cover’s ability to resist abrasion has been tested to handle 40,000 cycles.
## 15,000 cycles or more is suitable for furniture used every day at home. Over
## 30,000 cycles means a good ability to resist abrasion.The cover has a
## lightfastness level of 5-6 (the ability to resist colour fading) on a scale of
## 1 to 8. According to industry standards, a lightfastness level of 4 or higher
## is suitable for home use.", "193.254.57 The cover is easy to keep clean as it
## is removable and can be machine washed.", "392.873.98 Glass doors keep your
## favourite items free from dust but still visible.Adjustable shelves; adapt
## space between shelves according to your needs.Adjustable hinges allow you to
## adjust the door horizontally and vertically.This furniture must be fixed to the
## wall with the enclosed wall fastener.Handle with care! A damaged edge or
## scratched surface can cause the glass to suddenly crack and/or break. Avoid
## collisions from the side - this is where the glass is most vulnerable.Different
## wall materials require different types of fixing devices. Use fixing devices
## suitable for the walls in your home, sold separately.Min. ceiling height
## required: 205 cm.1 fixed shelf and 4 adjustable shelves included.May be
## completed with BILLY height extension unit in the same width for added storage
## vertically.", "404.728.61 Height adjustable armchair which you can swivel to
## the desired height.Slim lines, easy to place.You sit comfortably since the
## chair is adjustable in height.The safety castors have a pressure-sensitive
## brake mechanism that keeps the chair in place when you stand up, and releases
## automatically when you sit down.This product has been developed and tested for
## domestic use.", "602.957.06 With a media shelf you can make the most of the
## wall area, while freeing up space on the floor.Different wall materials require
## different types of fixing devices. Use fixing devices suitable for the walls in
## your home, sold separately.", "704.655.38 You sit comfortably thanks to the
## restful flexibility of the seat.You sit comfortably thanks to the padded
## seat.Velvet.The velvet reflects light in a characteristic way which may make
## the colour appear as if it changes.", "802.945.03 It’s easy to keep the cables
## from your TV and other devices out of sight but close at hand, as there are
## several cable outlets at the back of the TV bench.You can choose to stand the
## TV bench on the floor or mount it on the wall to free up floor space.If you
## want to organise inside you can complement with BESTÅ interior fittings.Steady
## also on uneven floors, thanks to the adjustable feet.This furniture must be
## fixed to the wall with the enclosed wall fastener.This TV bench can take a max
## load of 50 kg on the top.Different wall materials require different types of
## fixing devices. Use fixing devices suitable for the walls in your home, sold
## separately.May be completed with STALLARP, STUBBARP or NANNARP legs. This TV
## bench requires 4 legs and 1 BESTÅ supporting leg.May be completed with SULARP
## legs. This TV bench requires 2 legs and 1 BESTÅ supporting leg.", "904.710.86
## The chair legs are made of solid wood, which is a durable natural material.You
## sit comfortably thanks to the high back and seat with polyester wadding.For
## increased stability, re-tighten the screws about two weeks after assembly and
## when necessary.This chair has been tested for home use and meets the
## requirements for durability and safety, set forth in the following standards:
## EN 12520 and EN 1022.", "Carina Bengs/IKEA of Sweden", "Ebba Strandmark/IKEA of
## Sweden/Ola Wihlborg/Ehlén Johansson", "Ehlén Johansson/Andreas Fredriksson/IKEA
## of Sweden", "Ehlén Johansson/Francis Cayouette/IKEA of Sweden", "Eva Lilja
## Löwenhielm/Jonas Hultqvist/IKEA of Sweden", "Francis Cayouette/IKEA of Sweden",
## "Francis Cayouette/IKEA of Sweden/Ehlén Johansson", "IKEA of Sweden/Carina
## Bengs", "IKEA of Sweden/Ehlén Johansson/Ebba Strandmark", "Lisa Hilland",
## "Magnus Elebäck", "Noboru Nakamura/IKEA of Sweden"]
predictions_tbl <- predictions %>%
    as_tibble()

predictions_tbl %>%
    bind_cols(test_tbl)
## # A tibble: 475 × 14
##    predict   FALSE. TRUE.  item_id name       category     price sellable_online
##    <fct>      <dbl> <dbl>    <dbl> <fct>      <fct>        <dbl> <lgl>          
##  1 TRUE    0.000277 1.000 80155205 STIG       Bar furnitu…  4.23 TRUE           
##  2 TRUE    0.000277 1.000 30180504 NORBERG    Bar furnitu…  5.42 TRUE           
##  3 TRUE    0.000277 1.000 10122647 INGOLF     Bar furnitu…  5.84 TRUE           
##  4 TRUE    0.000277 1.000   121766 INGOLF     Bar furnitu…  5.98 TRUE           
##  5 TRUE    0.000277 1.000   397736 NORRARYD   Bar furnitu…  5.98 TRUE           
##  6 TRUE    0.000277 1.000 50420329 FREKVENS   Bar furnitu…  5.18 TRUE           
##  7 TRUE    0.000277 1.000 70246089 JANINGE    Bar furnitu…  6.39 TRUE           
##  8 TRUE    0.000277 1.000 30352246 RÅSKOG     Bar furnitu…  5.16 TRUE           
##  9 FALSE   0.0771   0.923 10400540 EKEDALEN   Bar furnitu…  5.84 TRUE           
## 10 TRUE    0.000277 1.000 90319918 HENRIKSDAL Bar furnitu…  6.54 TRUE           
## # ℹ 465 more rows
## # ℹ 6 more variables: other_colors <fct>, short_description <fct>,
## #   designer <fct>, depth <dbl>, height <dbl>, width <dbl>

Evaluate model

#h2o.performance
performance_h2o <- h2o.performance(best_model, newdata = test_h2o)
typeof(performance_h2o)
## [1] "S4"
slotNames(performance_h2o)
## [1] "algorithm" "on_train"  "on_valid"  "on_xval"   "metrics"
performance_h2o@metrics
## $model
## $model$`__meta`
## $model$`__meta`$schema_version
## [1] 3
## 
## $model$`__meta`$schema_name
## [1] "ModelKeyV3"
## 
## $model$`__meta`$schema_type
## [1] "Key<Model>"
## 
## 
## $model$name
## [1] "GBM_4_AutoML_1_20260423_105606"
## 
## $model$type
## [1] "Key<Model>"
## 
## $model$URL
## [1] "/3/Models/GBM_4_AutoML_1_20260423_105606"
## 
## 
## $model_checksum
## [1] "7802604689365284454"
## 
## $frame
## $frame$name
## [1] "test_tbl_sid_ad06_3"
## 
## 
## $frame_checksum
## [1] "-1254967299183438941"
## 
## $description
## NULL
## 
## $scoring_time
## [1] 1.77698e+12
## 
## $predictions
## NULL
## 
## $MSE
## [1] 0.006312253
## 
## $RMSE
## [1] 0.07944969
## 
## $nobs
## [1] 475
## 
## $custom_metric_name
## NULL
## 
## $custom_metric_value
## [1] 0
## 
## $r2
## [1] -0.005792482
## 
## $logloss
## [1] 0.04432942
## 
## $AUC
## [1] 0.9812853
## 
## $pr_auc
## [1] 0.9998785
## 
## $Gini
## [1] 0.9625706
## 
## $mean_per_class_error
## [1] 0.5
## 
## $domain
## [1] "FALSE" "TRUE" 
## 
## $cm
## $cm$`__meta`
## $cm$`__meta`$schema_version
## [1] 3
## 
## $cm$`__meta`$schema_name
## [1] "ConfusionMatrixV3"
## 
## $cm$`__meta`$schema_type
## [1] "ConfusionMatrix"
## 
## 
## $cm$table
## Confusion Matrix: Row labels: Actual class; Column labels: Predicted class
##        FALSE TRUE  Error      Rate
## FALSE      0    3 1.0000 =   3 / 3
## TRUE       0  472 0.0000 = 0 / 472
## Totals     0  475 0.0063 = 3 / 475
## 
## 
## $thresholds_and_metric_scores
## Metrics for Thresholds: Binomial metrics as a function of classification thresholds
##   threshold       f1       f2 f0point5 accuracy precision   recall specificity
## 1  0.999843 0.004228 0.002647 0.010504 0.008421  1.000000 0.002119    1.000000
## 2  0.999828 0.008439 0.005291 0.020833 0.010526  1.000000 0.004237    1.000000
## 3  0.999814 0.012632 0.007932 0.030992 0.012632  1.000000 0.006356    1.000000
## 4  0.999809 0.016807 0.010571 0.040984 0.014737  1.000000 0.008475    1.000000
## 5  0.999809 0.020964 0.013207 0.050813 0.016842  1.000000 0.010593    1.000000
##   absolute_mcc min_per_class_accuracy mean_per_class_accuracy tns fns fps tps
## 1     0.003662               0.002119                0.501059   3 471   0   1
## 2     0.005184               0.004237                0.502119   3 470   0   2
## 3     0.006356               0.006356                0.503178   3 469   0   3
## 4     0.007347               0.008475                0.504237   3 468   0   4
## 5     0.008223               0.010593                0.505297   3 467   0   5
##        tnr      fnr      fpr      tpr idx
## 1 1.000000 0.997881 0.000000 0.002119   0
## 2 1.000000 0.995763 0.000000 0.004237   1
## 3 1.000000 0.993644 0.000000 0.006356   2
## 4 1.000000 0.991525 0.000000 0.008475   3
## 5 1.000000 0.989407 0.000000 0.010593   4
## 
## ---
##     threshold       f1       f2 f0point5 accuracy precision   recall
## 207  0.999362 0.994687 0.992787 0.996593 0.989474  0.997868 0.991525
## 208  0.999358 0.995754 0.994487 0.997024 0.991579  0.997872 0.993644
## 209  0.999349 0.996819 0.996185 0.997453 0.993684  0.997877 0.995763
## 210  0.998378 0.996825 0.997459 0.996193 0.993684  0.995772 0.997881
## 211  0.998073 0.995772 0.997036 0.994510 0.991579  0.993671 0.997881
## 212  0.922942 0.996832 0.998730 0.994941 0.993684  0.993684 1.000000
##     specificity absolute_mcc min_per_class_accuracy mean_per_class_accuracy tns
## 207    0.666667     0.466898               0.666667                0.829096   2
## 208    0.666667     0.512562               0.666667                0.830155   2
## 209    0.666667     0.574289               0.666667                0.831215   2
## 210    0.333333     0.405224               0.333333                0.665607   1
## 211    0.000000     0.003662               0.000000                0.498941   0
## 212    0.000000     0.000000               0.000000                0.500000   0
##     fns fps tps      tnr      fnr      fpr      tpr idx
## 207   4   1 468 0.666667 0.008475 0.333333 0.991525 206
## 208   3   1 469 0.666667 0.006356 0.333333 0.993644 207
## 209   2   1 470 0.666667 0.004237 0.333333 0.995763 208
## 210   1   2 471 0.333333 0.002119 0.666667 0.997881 209
## 211   1   3 471 0.000000 0.002119 1.000000 0.997881 210
## 212   0   3 472 0.000000 0.000000 1.000000 1.000000 211
## 
## $max_criteria_and_metric_scores
## Maximum Metrics: Maximum metrics at their respective thresholds
##                         metric threshold      value idx
## 1                       max f1  0.922942   0.996832 211
## 2                       max f2  0.922942   0.998730 211
## 3                 max f0point5  0.999349   0.997453 208
## 4                 max accuracy  0.999349   0.993684 208
## 5                max precision  0.999843   1.000000   0
## 6                   max recall  0.922942   1.000000 211
## 7              max specificity  0.999843   1.000000   0
## 8             max absolute_mcc  0.999349   0.574289 208
## 9   max min_per_class_accuracy  0.999722   0.949153 186
## 10 max mean_per_class_accuracy  0.999722   0.974576 186
## 11                     max tns  0.999843   3.000000   0
## 12                     max fns  0.999843 471.000000   0
## 13                     max fps  0.998073   3.000000 210
## 14                     max tps  0.922942 472.000000 211
## 15                     max tnr  0.999843   1.000000   0
## 16                     max fnr  0.999843   0.997881   0
## 17                     max fpr  0.998073   1.000000 210
## 18                     max tpr  0.922942   1.000000 211
## 
## $gains_lift_table
## Gains/Lift Table: Avg response rate: 99.37 %, avg score: 99.96 %
##    group cumulative_data_fraction lower_threshold     lift cumulative_lift
## 1      1               0.01052632        0.999801 1.006356        1.006356
## 2      2               0.02315789        0.999778 1.006356        1.006356
## 3      3               0.03157895        0.999776 1.006356        1.006356
## 4      4               0.04000000        0.999775 1.006356        1.006356
## 5      5               0.05263158        0.999775 1.006356        1.006356
## 6      6               0.10105263        0.999775 1.006356        1.006356
## 7      7               0.15368421        0.999775 1.006356        1.006356
## 8      8               0.20000000        0.999775 1.006356        1.006356
## 9      9               0.30315789        0.999775 1.006356        1.006356
## 10    10               0.69894737        0.999775 1.006356        1.006356
## 11    11               0.69894737        0.999775 0.000000        1.006356
## 12    12               0.80000000        0.999775 1.006356        1.006356
## 13    13               0.89894737        0.999727 1.006356        1.006356
## 14    14               1.00000000        0.922942 0.943459        1.000000
##    response_rate    score cumulative_response_rate cumulative_score
## 1       1.000000 0.999820                 1.000000         0.999820
## 2       1.000000 0.999788                 1.000000         0.999803
## 3       1.000000 0.999776                 1.000000         0.999796
## 4       1.000000 0.999776                 1.000000         0.999791
## 5       1.000000 0.999775                 1.000000         0.999788
## 6       1.000000 0.999775                 1.000000         0.999782
## 7       1.000000 0.999775                 1.000000         0.999780
## 8       1.000000 0.999775                 1.000000         0.999779
## 9       1.000000 0.999775                 1.000000         0.999778
## 10      1.000000 0.999775                 1.000000         0.999776
## 11      0.000000 0.000000                 1.000000         0.999776
## 12      1.000000 0.999775                 1.000000         0.999776
## 13      1.000000 0.999759                 1.000000         0.999774
## 14      0.937500 0.997991                 0.993684         0.999594
##    capture_rate cumulative_capture_rate        gain cumulative_gain
## 1      0.010593                0.010593    0.635593        0.635593
## 2      0.012712                0.023305    0.635593        0.635593
## 3      0.008475                0.031780    0.635593        0.635593
## 4      0.008475                0.040254    0.635593        0.635593
## 5      0.012712                0.052966    0.635593        0.635593
## 6      0.048729                0.101695    0.635593        0.635593
## 7      0.052966                0.154661    0.635593        0.635593
## 8      0.046610                0.201271    0.635593        0.635593
## 9      0.103814                0.305085    0.635593        0.635593
## 10     0.398305                0.703390    0.635593        0.635593
## 11     0.000000                0.703390 -100.000000        0.635593
## 12     0.101695                0.805085    0.635593        0.635593
## 13     0.099576                0.904661    0.635593        0.635593
## 14     0.095339                1.000000   -5.654131        0.000000
##    kolmogorov_smirnov
## 1            0.010593
## 2            0.023305
## 3            0.031780
## 4            0.040254
## 5            0.052966
## 6            0.101695
## 7            0.154661
## 8            0.201271
## 9            0.305085
## 10           0.703390
## 11           0.703390
## 12           0.805085
## 13           0.904661
## 14           0.000000
h2o.auc(performance_h2o)
## [1] 0.9812853
h2o.confusionMatrix(performance_h2o)
## Confusion Matrix (vertical: actual; across: predicted)  for max f1 @ threshold = 0.922942127589153:
##        FALSE TRUE    Error    Rate
## FALSE      0    3 1.000000    =3/3
## TRUE       0  472 0.000000  =0/472
## Totals     0  475 0.006316  =3/475
h2o.metric(performance_h2o)
## Metrics for Thresholds: Binomial metrics as a function of classification thresholds
##   threshold       f1       f2 f0point5 accuracy precision   recall specificity
## 1  0.999843 0.004228 0.002647 0.010504 0.008421  1.000000 0.002119    1.000000
## 2  0.999828 0.008439 0.005291 0.020833 0.010526  1.000000 0.004237    1.000000
## 3  0.999814 0.012632 0.007932 0.030992 0.012632  1.000000 0.006356    1.000000
## 4  0.999809 0.016807 0.010571 0.040984 0.014737  1.000000 0.008475    1.000000
## 5  0.999809 0.020964 0.013207 0.050813 0.016842  1.000000 0.010593    1.000000
##   absolute_mcc min_per_class_accuracy mean_per_class_accuracy tns fns fps tps
## 1     0.003662               0.002119                0.501059   3 471   0   1
## 2     0.005184               0.004237                0.502119   3 470   0   2
## 3     0.006356               0.006356                0.503178   3 469   0   3
## 4     0.007347               0.008475                0.504237   3 468   0   4
## 5     0.008223               0.010593                0.505297   3 467   0   5
##        tnr      fnr      fpr      tpr idx
## 1 1.000000 0.997881 0.000000 0.002119   0
## 2 1.000000 0.995763 0.000000 0.004237   1
## 3 1.000000 0.993644 0.000000 0.006356   2
## 4 1.000000 0.991525 0.000000 0.008475   3
## 5 1.000000 0.989407 0.000000 0.010593   4
## 
## ---
##     threshold       f1       f2 f0point5 accuracy precision   recall
## 207  0.999362 0.994687 0.992787 0.996593 0.989474  0.997868 0.991525
## 208  0.999358 0.995754 0.994487 0.997024 0.991579  0.997872 0.993644
## 209  0.999349 0.996819 0.996185 0.997453 0.993684  0.997877 0.995763
## 210  0.998378 0.996825 0.997459 0.996193 0.993684  0.995772 0.997881
## 211  0.998073 0.995772 0.997036 0.994510 0.991579  0.993671 0.997881
## 212  0.922942 0.996832 0.998730 0.994941 0.993684  0.993684 1.000000
##     specificity absolute_mcc min_per_class_accuracy mean_per_class_accuracy tns
## 207    0.666667     0.466898               0.666667                0.829096   2
## 208    0.666667     0.512562               0.666667                0.830155   2
## 209    0.666667     0.574289               0.666667                0.831215   2
## 210    0.333333     0.405224               0.333333                0.665607   1
## 211    0.000000     0.003662               0.000000                0.498941   0
## 212    0.000000     0.000000               0.000000                0.500000   0
##     fns fps tps      tnr      fnr      fpr      tpr idx
## 207   4   1 468 0.666667 0.008475 0.333333 0.991525 206
## 208   3   1 469 0.666667 0.006356 0.333333 0.993644 207
## 209   2   1 470 0.666667 0.004237 0.333333 0.995763 208
## 210   1   2 471 0.333333 0.002119 0.666667 0.997881 209
## 211   1   3 471 0.000000 0.002119 1.000000 0.997881 210
## 212   0   3 472 0.000000 0.000000 1.000000 1.000000 211