Apply it to your data 4: Improved
Nils Skogestig
2025-02-08
Click here for the data. Goal: Predict future price of IKEA furniture
Import data
ikea <- readr::read_csv('https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2020/2020-11-03/ikea.csv')## New names:
## Rows: 3694 Columns: 14
## ── Column specification
## ──────────────────────────────────────────────────────── Delimiter: "," chr
## (7): name, category, old_price, link, other_colors, short_description, d... dbl
## (6): ...1, item_id, price, depth, height, width lgl (1): sellable_online
## ℹ Use `spec()` to retrieve the full column specification for this data. ℹ
## Specify the column types or set `show_col_types = FALSE` to quiet this message.
## • `` -> `...1`skimr::skim(ikea)| Name | ikea |
| Number of rows | 3694 |
| Number of columns | 14 |
| _______________________ | |
| Column type frequency: | |
| character | 7 |
| logical | 1 |
| numeric | 6 |
| ________________________ | |
| Group variables | None |
Variable type: character
| skim_variable | n_missing | complete_rate | min | max | empty | n_unique | whitespace |
|---|---|---|---|---|---|---|---|
| name | 0 | 1 | 3 | 27 | 0 | 607 | 0 |
| category | 0 | 1 | 4 | 36 | 0 | 17 | 0 |
| old_price | 0 | 1 | 4 | 13 | 0 | 365 | 0 |
| link | 0 | 1 | 52 | 163 | 0 | 2962 | 0 |
| other_colors | 0 | 1 | 2 | 3 | 0 | 2 | 0 |
| short_description | 0 | 1 | 3 | 63 | 0 | 1706 | 0 |
| designer | 0 | 1 | 3 | 1261 | 0 | 381 | 0 |
Variable type: logical
| skim_variable | n_missing | complete_rate | mean | count |
|---|---|---|---|---|
| sellable_online | 0 | 1 | 0.99 | TRU: 3666, FAL: 28 |
Variable type: numeric
| skim_variable | n_missing | complete_rate | mean | sd | p0 | p25 | p50 | p75 | p100 | hist |
|---|---|---|---|---|---|---|---|---|---|---|
| …1 | 0 | 1.00 | 1846.50 | 1066.51 | 0 | 923.25 | 1846.5 | 2769.75 | 3693 | ▇▇▇▇▇ |
| item_id | 0 | 1.00 | 48632396.79 | 28887094.10 | 58487 | 20390574.00 | 49288078.0 | 70403572.75 | 99932615 | ▇▇▇▇▇ |
| price | 0 | 1.00 | 1078.21 | 1374.65 | 3 | 180.90 | 544.7 | 1429.50 | 9585 | ▇▁▁▁▁ |
| depth | 1463 | 0.60 | 54.38 | 29.96 | 1 | 38.00 | 47.0 | 60.00 | 257 | ▇▃▁▁▁ |
| height | 988 | 0.73 | 101.68 | 61.10 | 1 | 67.00 | 83.0 | 124.00 | 700 | ▇▂▁▁▁ |
| width | 589 | 0.84 | 104.47 | 71.13 | 1 | 60.00 | 80.0 | 140.00 | 420 | ▇▅▂▁▁ |
data <- ikea %>%
# Handle missing values
filter(!is.na(height), !is.na(width), !is.na(depth)) %>%
# Convert all character variables to factors for machine learning compatibility
mutate(across(where(is.character), as.factor)) %>%
mutate(across(is.logical, as.factor)) %>%
# Handle multiple designers by splitting them into separate rows
separate_rows(designer, sep = "/") %>%
# Remove unnecessary columns
select(-...1, -link, -old_price, -designer, -name) %>%
# Log transform price to address skewness
mutate(price = log(price))## Warning: There was 1 warning in `mutate()`.
## ℹ In argument: `across(is.logical, as.factor)`.
## Caused by warning:
## ! Use of bare predicate functions was deprecated in tidyselect 1.1.0.
## ℹ Please use wrap predicates in `where()` instead.
## # Was:
## data %>% select(is.logical)
##
## # Now:
## data %>% select(where(is.logical))Explore Data - Identifying good predictors
# Step 1: Prepare data by binarising data
binarized_table <- data %>%
select(-item_id, -short_description) %>%
binarize()
binarized_table %>% glimpse()## Rows: 2,714
## Columns: 35
## $ category__Bar_furniture <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ category__Beds <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Bookcases_&_shelving_units` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Cabinets_&_cupboards` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category__Café_furniture <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category__Chairs <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Chests_of_drawers_&_drawer_units` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Children's_furniture` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category__Nursery_furniture <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category__Outdoor_furniture <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Sofas_&_armchairs` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__Tables_&_desks` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__TV_&_media_furniture` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category__Wardrobes <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `category__-OTHER` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `price__-Inf_5.84354441703136` <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ price__5.84354441703136_6.75693238924755 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ price__6.75693238924755_7.52023455647463 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ price__7.52023455647463_Inf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ sellable_online__TRUE <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ `sellable_online__-OTHER` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ other_colors__No <dbl> 0, 1, 1, 1, 1, 1, 1, 1, 1…
## $ other_colors__Yes <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `depth__-Inf_40` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ depth__40_48 <dbl> 0, 0, 1, 1, 1, 1, 1, 1, 1…
## $ depth__48_66 <dbl> 1, 1, 0, 0, 0, 0, 0, 0, 0…
## $ depth__66_Inf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `height__-Inf_74` <dbl> 0, 1, 0, 0, 0, 0, 0, 0, 0…
## $ height__74_95 <dbl> 0, 0, 1, 1, 1, 1, 1, 0, 0…
## $ height__95_180.75 <dbl> 1, 0, 0, 0, 0, 0, 0, 1, 1…
## $ height__180.75_Inf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `width__-Inf_60` <dbl> 1, 0, 1, 1, 1, 1, 1, 1, 1…
## $ width__60_100 <dbl> 0, 1, 0, 0, 0, 0, 0, 0, 0…
## $ width__100_180 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ width__180_Inf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0…# Correlate
corr_tbl <- binarized_table %>%
correlate(price__7.52023455647463_Inf)
corr_tbl## # A tibble: 35 × 3
## feature bin correlation
## <fct> <chr> <dbl>
## 1 price 7.52023455647463_Inf 1
## 2 width 180_Inf 0.610
## 3 depth 66_Inf 0.418
## 4 width -Inf_60 -0.356
## 5 category Sofas_&_armchairs 0.343
## 6 price -Inf_5.84354441703136 -0.335
## 7 price 6.75693238924755_7.52023455647463 -0.333
## 8 price 5.84354441703136_6.75693238924755 -0.332
## 9 category Wardrobes 0.302
## 10 height -Inf_74 -0.283
## # ℹ 25 more rows# Step 3: Plot Correlation Funnel
corr_tbl %>%
plot_correlation_funnel()## Warning: ggrepel: 12 unlabeled data points (too many overlaps). Consider
## increasing max.overlaps
Preprocessing - Recipe Improvements
ikea_recipe_improved <- recipe(price ~ ., data = data) %>%
update_role(item_id, new_role = "id variable") %>%
step_tokenize(short_description) %>%
step_tokenfilter(short_description, max_tokens = 150) %>% # Adjust max_tokens
step_tfidf(short_description) %>%
step_other(category, threshold = 0.05) %>% # More aggressive threshold
step_novel(all_nominal_predictors()) %>%
step_dummy(all_nominal_predictors(), one_hot = TRUE) %>%
step_YeoJohnson(all_numeric_predictors()) # Normalize skewed variables
ikea_recipe_improved %>% prep() %>% juice() %>% glimpse()## Rows: 2,714
## Columns: 170
## $ item_id <dbl> 80155205, 30180504, 10122647…
## $ depth <dbl> 4.415739, 4.641769, 4.286541…
## $ height <dbl> 13.841936, 9.122162, 13.2276…
## $ width <dbl> 6.309291, 6.780073, 5.455728…
## $ price <dbl> 4.234107, 5.416100, 5.843544…
## $ tfidf_short_description_1 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_10 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_100x60x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_120 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_120x42x240 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_120x42x74 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_140x200 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_143 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_147x147 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_150x44x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_150x58x201 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_150x60x201 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_150x60x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_150x66x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_160x30x202 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_160x30x237 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_165x55x216 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_180x42x74 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_2 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_200x44x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_200x60x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_200x66x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_223x80x84 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_250x60x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_25x51x70 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_3 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_300x60x236 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_35x35x35 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_4 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_40x30x106 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_41x101 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_41x61 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_5 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_6 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_60x22x202 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_60x50x128 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_60x50x192 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_61x101 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_63 <dbl> 0.0000000, 0.0000000, 0.8787…
## $ tfidf_short_description_74 <dbl> 0.7582519, 0.0000000, 0.0000…
## $ tfidf_short_description_75 <dbl> 0.0000000, 0.0000000, 0.0000…
## $ tfidf_short_description_76 <dbl> 0.000000, 0.000000, 0.000000…
## $ tfidf_short_description_77x147 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_8 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_80x200 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_80x30x202 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_89x30x124 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_90 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_90x200 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_90x79x102 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_99x44x56 <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_add <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_and <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_armchair <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_armrest <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_armrests <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_backrest <dbl> 0.6416913, 0.0000000, 0.6416…
## $ tfidf_short_description_backrests <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_bar <dbl> 0.5823944, 0.0000000, 0.5823…
## $ tfidf_short_description_baskets <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_bed <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_bedside <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_bench <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_bookcase <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_box <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_cabinet <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_cabinets <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_castors <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_chair <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_chaise <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_changing <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_chest <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ `tfidf_short_description_children's` <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_clothes <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_cm <dbl> 0.12453381, 0.17345325, 0.12…
## $ tfidf_short_description_combination <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_corner <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_cover <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_cushion <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_day <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_desk <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_display <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_door <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_doors <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_drawer <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_drawers <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_dressing <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_drs <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_easy <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_extension <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_feet <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_foldable <dbl> 0.0000000, 0.0000000, 0.0000…
## $ tfidf_short_description_folding <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_footstool <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_for <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_frame <dbl> 0.0000000, 0.0000000, 0.0000…
## $ tfidf_short_description_glass <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_headboard <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_high <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_highchair <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_in <dbl> 0.0000000, 0.0000000, 0.0000…
## $ tfidf_short_description_inserts <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_junior <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_leg <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_legs <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_lock <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_longue <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_mattresses <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_media <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_mesh <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_modular <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_module <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_mounted <dbl> 0.0000000, 0.9667203, 0.0000…
## $ tfidf_short_description_of <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_on <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_outdoor <dbl> 0.0000000, 0.0000000, 0.0000…
## $ tfidf_short_description_panel <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_plinth <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_rail <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_rails <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_seat <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_section <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_sections <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_shelf <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_shelves <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_shelving <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_side <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_sliding <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_smart <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_sofa <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_step <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_stool <dbl> 0.5269921, 0.0000000, 0.5269…
## $ tfidf_short_description_storage <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_table <dbl> 0.000000, 0.950408, 0.000000…
## $ tfidf_short_description_three <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_top <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_toy <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_tray <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_triple <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_tv <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_two <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_underframe <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_unit <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_upright <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_ut <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_w <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_wall <dbl> 0.0000000, 0.8271747, 0.0000…
## $ tfidf_short_description_wardrobe <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_wire <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ tfidf_short_description_with <dbl> 0.2918635, 0.0000000, 0.2918…
## $ category_Beds <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Bookcases...shelving.units <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Cabinets...cupboards <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Chairs <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Chests.of.drawers...drawer.units <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Sofas...armchairs <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_Wardrobes <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ category_other <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ category_new <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ sellable_online_FALSE. <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ sellable_online_TRUE. <dbl> 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ sellable_online_new <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ other_colors_No <dbl> 0, 1, 1, 1, 1, 1, 1, 1, 1, 1…
## $ other_colors_Yes <dbl> 1, 0, 0, 0, 0, 0, 0, 0, 0, 0…
## $ other_colors_new <dbl> 0, 0, 0, 0, 0, 0, 0, 0, 0, 0…Data Splitting and Cross-Validation Setup
set.seed(1234)
ikea_split <- initial_split(data, prop = 0.75)
ikea_train <- training(ikea_split)
ikea_test <- testing(ikea_split)
set.seed(2345)
ikea_cv <- vfold_cv(ikea_train, v = 5)Model Specification (XGBoost, Random Forest, SVM)
# XGBoost Model
xgboost_spec <- boost_tree(trees = tune(), min_n = tune(), mtry = tune(), learn_rate = tune()) %>%
set_mode("regression") %>%
set_engine("xgboost")
# Random Forest Model
rf_spec <- rand_forest(trees = tune(), min_n = tune(), mtry = tune()) %>%
set_mode("regression") %>%
set_engine("ranger")
# SVM Model
svm_spec <- svm_rbf(cost = tune(), rbf_sigma = tune()) %>%
set_mode("regression") %>%
set_engine("kernlab")
# Create workflows
xgboost_workflow <- workflow() %>%
add_recipe(ikea_recipe_improved) %>%
add_model(xgboost_spec)
rf_workflow <- workflow() %>%
add_recipe(ikea_recipe_improved) %>%
add_model(rf_spec)
svm_workflow <- workflow() %>%
add_recipe(ikea_recipe_improved) %>%
add_model(svm_spec)Hyperparamter Tuning and Grid Search
# Finalize mtry based on training data
mtry_range <- finalize(mtry(), x = ikea_train)
# XGBoost Grid
xgboost_grid <- grid_latin_hypercube(trees(), min_n(), mtry = mtry_range, learn_rate(), size = 20)## Warning: `grid_latin_hypercube()` was deprecated in dials 1.3.0.
## ℹ Please use `grid_space_filling()` instead.
## This warning is displayed once every 8 hours.
## Call `lifecycle::last_lifecycle_warnings()` to see where this warning was
## generated.# Random Forest Grid
rf_grid <- grid_regular(trees(), min_n(), mtry = mtry_range, levels = 5)
# SVM Grid (SVM doesn't use mtry, so it remains the same)
svm_grid <- grid_regular(cost(), rbf_sigma(), levels = 5)
# Tune XGBoost
set.seed(3456)
xgboost_tune <- tune_grid(xgboost_workflow, resamples = ikea_cv, grid = xgboost_grid)## → A | warning: A correlation computation is required, but `estimate` is constant and has 0
## standard deviation, resulting in a divide by 0 error. `NA` will be returned.## There were issues with some computations A: x1There were issues with some computations A: x2There were issues with some computations A: x3There were issues with some computations A: x4There were issues with some computations A: x5There were issues with some computations A: x5# Tune Random Forest
set.seed(4567)
rf_tune <- tune_grid(rf_workflow, resamples = ikea_cv, grid = rf_grid)
# Tune SVM
set.seed(5678)
svm_tune <- tune_grid(svm_workflow, resamples = ikea_cv, grid = svm_grid)## → A | warning: Variable(s) `' constant. Cannot scale data.## There were issues with some computations A: x1There were issues with some computations A: x2There were issues with some computations A: x3There were issues with some computations A: x4There were issues with some computations A: x5There were issues with some computations A: x6There were issues with some computations A: x7There were issues with some computations A: x8There were issues with some computations A: x9There were issues with some computations A: x10There were issues with some computations A: x11There were issues with some computations A: x12There were issues with some computations A: x13There were issues with some computations A: x14There were issues with some computations A: x15There were issues with some computations A: x16There were issues with some computations A: x17There were issues with some computations A: x18There were issues with some computations A: x19There were issues with some computations A: x20There were issues with some computations A: x21There were issues with some computations A: x22There were issues with some computations A: x23There were issues with some computations A: x24There were issues with some computations A: x25There were issues with some computations A: x26There were issues with some computations A: x27There were issues with some computations A: x28There were issues with some computations A: x29There were issues with some computations A: x30There were issues with some computations A: x31There were issues with some computations A: x32There were issues with some computations A: x33There were issues with some computations A: x34There were issues with some computations A: x35There were issues with some computations A: x36There were issues with some computations A: x37There were issues with some computations A: x38There were issues with some computations A: x39There were issues with some computations A: x40There were issues with some computations A: x41There were issues with some computations A: x42There were issues with some computations A: x43There were issues with some computations A: x44There were issues with some computations A: x45There were issues with some computations A: x46There were issues with some computations A: x47There were issues with some computations A: x48There were issues with some computations A: x49There were issues with some computations A: x50There were issues with some computations A: x51There were issues with some computations A: x52There were issues with some computations A: x53There were issues with some computations A: x54There were issues with some computations A: x55There were issues with some computations A: x56There were issues with some computations A: x57There were issues with some computations A: x58There were issues with some computations A: x59There were issues with some computations A: x60There were issues with some computations A: x61There were issues with some computations A: x62There were issues with some computations A: x63There were issues with some computations A: x64There were issues with some computations A: x65There were issues with some computations A: x66There were issues with some computations A: x67There were issues with some computations A: x68There were issues with some computations A: x69There were issues with some computations A: x70There were issues with some computations A: x71There were issues with some computations A: x72There were issues with some computations A: x73There were issues with some computations A: x74There were issues with some computations A: x75There were issues with some computations A: x76There were issues with some computations A: x77There were issues with some computations A: x78There were issues with some computations A: x79There were issues with some computations A: x80There were issues with some computations A: x81There were issues with some computations A: x82There were issues with some computations A: x83There were issues with some computations A: x84There were issues with some computations A: x85There were issues with some computations A: x86There were issues with some computations A: x87There were issues with some computations A: x88There were issues with some computations A: x89There were issues with some computations A: x90There were issues with some computations A: x91There were issues with some computations A: x92There were issues with some computations A: x93There were issues with some computations A: x94There were issues with some computations A: x95There were issues with some computations A: x96There were issues with some computations A: x97There were issues with some computations A: x98There were issues with some computations A: x99There were issues with some computations A: x100There were issues with some computations A: x101There were issues with some computations A: x102There were issues with some computations A: x103There were issues with some computations A: x104There were issues with some computations A: x105There were issues with some computations A: x106There were issues with some computations A: x107There were issues with some computations A: x108There were issues with some computations A: x109There were issues with some computations A: x110There were issues with some computations A: x111There were issues with some computations A: x112There were issues with some computations A: x113There were issues with some computations A: x114There were issues with some computations A: x115There were issues with some computations A: x116There were issues with some computations A: x117There were issues with some computations A: x118There were issues with some computations A: x119There were issues with some computations A: x120There were issues with some computations A: x121There were issues with some computations A: x122There were issues with some computations A: x123There were issues with some computations A: x124There were issues with some computations A: x125There were issues with some computations A: x125Evaluate Models
# Collect metrics for each model
xgboost_results <- collect_metrics(xgboost_tune)
rf_results <- collect_metrics(rf_tune)
svm_results <- collect_metrics(svm_tune)
# Compare RMSE and R-squared for all models
bind_rows(xgboost_results, rf_results, svm_results) %>%
filter(.metric == "rmse") %>%
arrange(mean)## # A tibble: 170 × 12
## mtry trees min_n learn_rate .metric .estimator mean n std_err .config
## <int> <int> <int> <dbl> <chr> <chr> <dbl> <int> <dbl> <chr>
## 1 8 1529 36 0.0720 rmse standard 0.387 5 0.00787 Preproce…
## 2 NA NA NA NA rmse standard 0.396 5 0.0125 Preproce…
## 3 9 466 11 0.0210 rmse standard 0.443 5 0.0116 Preproce…
## 4 9 1000 2 NA rmse standard 0.470 5 0.00942 Preproce…
## 5 9 500 2 NA rmse standard 0.470 5 0.0102 Preproce…
## 6 9 1500 2 NA rmse standard 0.471 5 0.0104 Preproce…
## 7 9 2000 2 NA rmse standard 0.471 5 0.0102 Preproce…
## 8 9 500 11 NA rmse standard 0.476 5 0.0102 Preproce…
## 9 9 1500 11 NA rmse standard 0.477 5 0.00988 Preproce…
## 10 9 2000 11 NA rmse standard 0.478 5 0.00943 Preproce…
## # ℹ 160 more rows
## # ℹ 2 more variables: cost <dbl>, rbf_sigma <dbl>Finalize best model and predict
# Select best performing model (e.g., XGBoost)
xgboost_best <- finalize_workflow(xgboost_workflow, select_best(xgboost_tune, metric = "rmse"))
# Fit the final model on the entire training set and test it
ikea_fit <- last_fit(xgboost_best, ikea_split)
# Evaluate on test data
test_metrics <- collect_metrics(ikea_fit)
test_predictions <- collect_predictions(ikea_fit)
# Plot actual vs predicted prices
test_predictions %>%
ggplot(aes(x = price, y = .pred)) +
geom_point(alpha = 0.3, color = "midnightblue") +
geom_abline(lty = 2, color = "gray50") +
coord_fixed() +
labs(title = "Predicted vs Actual Prices (Test Data)", x = "Actual Price (log-transformed)", y = "Predicted Price")
# Extract RMSE, R-squared, and variance explained
rmse_value <- test_metrics %>%
filter(.metric == "rmse") %>%
pull(".estimate") # Corrected: ".estimate" is the correct column
rsq_value <- test_metrics %>%
filter(.metric == "rsq") %>%
pull(".estimate") # Corrected: ".estimate" is the correct column
variance_explained <- rsq_value * 100
# Print the values
print(paste("RMSE:", rmse_value))## [1] "RMSE: 0.379546001213468"print(paste("R-squared:", rsq_value))## [1] "R-squared: 0.903283387235221"print(paste("Variance Explained (%):", variance_explained))## [1] "Variance Explained (%): 90.3283387235221"Conclusion of Result
The final model for predicting IKEA furniture prices performed very well, with an RMSE of 0.3795 and an R-squared of 0.9033. This means the model explains about 90.33% of the price variations, showing that it makes accurate predictions.
Several improvements helped boost performance, including:
- Fixing missing values using KNN imputation.
- Refining categories with one-hot encoding and novel detection.
- Using TF-IDF to better analyze short descriptions.
The low RMSE means the predicted prices are very close to actual prices, and the high R-squared shows the model captures important price patterns. While there is always room for small improvements, this model is rather reliable for predicting IKEA furniture prices.