basement_full <- full %>% 
  select(df_id, id, contains("bsmt"))

# define numeric and character column names
num_names <- c("df_id", basement_full %>% select_if(is.numeric) %>% names)
char_names <- c("id", basement_full %>% select_if(is.character) %>% names())

# filter observations in which there is at least 1 missing value in a numeric column
basement_full %>% 
  filter_at(.vars = num_names, any_vars(is.na(.))) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left") %>% 
  scroll_box(width = "800px")

# filter observations in which there is at least 1 missing value in a character column
basement_full %>% 
  filter_at(.vars = char_names, any_vars(is.na(.))) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left") %>% 
  scroll_box(width = "800px", height = "400px")

# Replace 'NA' in numeric variables with '0'
basement_full <- basement_full %>% 
  mutate_at(.vars = num_names[-(1:2)], .funs = replace_na, 0)

# Create a new dataframe, with only character variables :
# Replace 'NA' in character variables : only if all values are 'NA' on the same observation => 'None'
# Other 'NA' will remain 'NA' for now
basement_char <- basement_full %>% 
  select(char_names) %>% 
  filter_at(.vars = char_names[-(1:2)], all_vars(is.na(.))) %>%   # filter rows which have all character colomns to 'NA'
  map_df(.x = ., .f = ~replace_na(data = .x, replace = "None")) %>%   # in these rows only, replace 'NA' with 'None'
  mutate(id = as.numeric(id))

# Replace the corresponding rows in 'basement_full'
basement_full[basement_char$id, char_names] <- basement_char

# Let's check our results.
basement_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  mutate_all(function(x) {
    cell_spec(x, 
              background = ifelse(is.na(x), "orange", "default"),
              color = ifelse(is.na(x), "white", "default"))
  }) %>% 
  kable(format = "html", escape = F) %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left") %>% 
  scroll_box(width = "800px", height = "400px")

full[, colnames(basement_full)] <- basement_full

# Clean
rm(basement_full, basement_char, char_names, num_names)

pool_full <- full %>% 
  select(df_id, id, pool_area, pool_qc)

pool_full %>% 
  filter(is.na(pool_qc), pool_area != 0) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Change the missing values into 'None' for character variables
pool_full$pool_qc[is.na(pool_full$pool_qc) & pool_full$pool_area == 0] <- "None"

# Double check our results
pool_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  mutate(pool_qc = cell_spec(pool_qc, "html", 
                             color = ifelse(is.na(pool_qc), "white", "default"),
                             background = ifelse(is.na(pool_qc), "orange", "default"))) %>% 
  kable(escape = FALSE) %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

full[, colnames(pool_full)] <- pool_full

# Clean
rm(pool_full)

misc_full <- full %>% 
  select(df_id, id, contains("misc"))

misc_full %>% 
  filter(is.na(misc_feature), misc_val != 0) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Change the missing values into 'None' for character variables.
misc_full$misc_feature[is.na(misc_full$misc_feature) & misc_full$misc_val == 0] <- "None"

# Double check our results.
misc_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  mutate(misc_feature = cell_spec(misc_feature, "html", 
                                  color = ifelse(is.na(misc_feature), "white", "default"),
                                  background = ifelse(is.na(misc_feature), "orange", "default"))) %>% 
  kable(escape = FALSE) %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Copy our new values into the original dataset.
full[, colnames(misc_full)] <- misc_full

# Clean : remove 'misc_full'.
rm(misc_full)

garage_full <- full %>% 
  select(1:2, contains("garage"))

# missing value for all garage variables
garage_full %>% 
  select(-id, -df_id) %>% 
  sapply(function(x) sum(is.na(x)))

garage_full %>%
  filter(is.na(garage_type)) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left") %>% 
  scroll_box(width = "800px", height = "400px")

# transform the 'garage_yr_blt' as a character variable
garage_full$garage_yr_blt <- as.character(garage_full$garage_yr_blt)

# define numeric and character column names
num_names <- garage_full %>% select_if(is.numeric) %>% names()
char_names <- garage_full %>% select_if(is.character) %>% names()

# filter observations which have 'NA' in 'garage_type', and replace
# 'NA's with 0 for numerical and 'None' for character variables
garage_na_type <- garage_full %>% 
  filter(is.na(garage_type)) %>% 
  mutate_at(.vars = num_names, replace_na, 0) %>% 
  mutate_at(.vars = char_names, replace_na, "None")

# Replace the corresponding rows in 'garage_full'
garage_full[garage_na_type$id, ] <- garage_na_type

# Let's check our results.
garage_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left") %>% 
  scroll_box(width = "800px")

garage_full[garage_full$id == 2577, char_names[-1]] <- "None"
garage_full[garage_full$id == 2577, num_names[-1]] <- 0

# Copy our new values into the original dataset.
full[, colnames(garage_full)] <- garage_full

# Clean
rm(garage_full, garage_na_type, char_names, num_names)

fire_full <- full %>% 
  select(df_id, id, contains("fire"))

fire_full %>% 
  filter(is.na(fireplace_qu), fireplaces != 0) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Change the missing values into 'None' for character variables
fire_full$fireplace_qu[is.na(fire_full$fireplace_qu)] <- "None"

# Double check our results.
fire_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Copy our new values into the original dataset.
full[, colnames(fire_full)] <- fire_full

# Clean : remove 'fire_full'.
rm(fire_full)

mas_full <- full %>% 
  select(df_id, id, contains("mas"))

mas_full %>% 
  filter(is.na(mas_vnr_type), mas_vnr_area != 0) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Change the missing values into 'None' for character variables and '0' for numeric variables.
mas_full$mas_vnr_type[is.na(mas_full$mas_vnr_type) & is.na(mas_full$mas_vnr_area)] <- "None"
mas_full$mas_vnr_area[is.na(mas_full$mas_vnr_area)] <- 0

# Double check our results.
mas_full %>% 
  filter_all(any_vars(is.na(.))) %>% 
  mutate(mas_vnr_type = cell_spec(mas_vnr_type, "html",
                                  color = ifelse(is.na(mas_vnr_type), "white", "default"),
                                  background = ifelse(is.na(mas_vnr_type), "orange", "default"))) %>% 
  kable(escape = FALSE) %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# Copy our new values into the original dataset.
full[, colnames(mas_full)] <- mas_full

# Clean : remove 'mas_full'.
rm(mas_full)

full$alley[is.na(full$alley)] <- "None"
full$fence[is.na(full$fence)] <- "None"

full %>% 
  sapply(function(x) sum(is.na(x))) %>% 
  enframe(name = "variable", value = "missing_nb") %>% 
  arrange(-missing_nb) %>% 
  filter(missing_nb != 0, variable != "sale_price") %>% 
  mutate(missing_nb = color_bar("lightgreen")(missing_nb)) %>% 
  kable(escape = FALSE) %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

full$ms_sub_class <- as.character(full$ms_sub_class)
full$overall_qual <- as.character(full$overall_qual)
full$overall_cond <- as.character(full$overall_cond)

full <- full %>% 
  mutate_if(is.character, as.factor)

table(full$garage_yr_blt, useNA = "always")

full %>% 
  filter(garage_yr_blt == "2207") %>% 
  select(df_id, id, year_built, year_remod_add, garage_yr_blt) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

full$garage_yr_blt[full$garage_yr_blt == "2207"] <- "2007"
full$garage_yr_blt <- fct_drop(full$garage_yr_blt)

full %>% 
  select(df_id, id, year_built, year_remod_add, garage_yr_blt) %>% 
  filter(year_built > year_remod_add) %>% 
  kable() %>% 
  kable_styling(bootstrap_options = "striped", full_width = F, position = "left")

# change the 'year_remod_add' value
full$year_remod_add[full$id == 1877] <- 2002

# transform following features as ordinal variables
full$overall_qual <- ordered(x = full$overall_qual, levels = as.character(1:10))
full$overall_cond <- ordered(x = full$overall_cond, levels = as.character(1:10))

full$exter_qual <- ordered(x = full$exter_qual, levels = c("Po", "Fa", "TA", "Gd", "Ex"))
full$exter_cond <- ordered(x = full$exter_cond, levels = c("Po", "Fa", "TA", "Gd", "Ex"))

full$bsmt_qual <- ordered(x = full$bsmt_qual, levels = c("None", "Po", "Fa", "TA", "Gd", "Ex"))
full$bsmt_cond <- ordered(x = full$bsmt_cond, levels = c("None", "Po", "Fa", "TA", "Gd", "Ex"))
full$bsmt_fin_type1 <- ordered(x = full$bsmt_fin_type1, levels = c("None", "Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"))
full$bsmt_fin_type2 <- ordered(x = full$bsmt_fin_type2, levels = c("None", "Unf", "LwQ", "Rec", "BLQ", "ALQ", "GLQ"))

full$heating_qc <- ordered(x = full$heating_qc, levels = c("Po", "Fa", "TA", "Gd", "Ex"))

full$kitchen_qual <- ordered(x = full$kitchen_qual, levels = c("Po", "Fa", "TA", "Gd", "Ex"))

full$functional <- ordered(x = full$functional, levels = c("Sal", "Sev", "Maj2", "Maj1", "Mod", "Min2", "Min1", "Typ"))

full$fireplace_qu <- ordered(x = full$fireplace_qu, levels = c("None", "Po", "Fa", "TA", "Gd", "Ex"))

full$garage_qual <- ordered(x = full$garage_qual, levels = c("None", "Po", "Fa", "TA", "Gd", "Ex"))
full$garage_cond <- ordered(x = full$garage_cond, levels = c("None", "Po", "Fa", "TA", "Gd", "Ex"))

full$pool_qc <- ordered(x = full$pool_qc, levels = c("None", "Fa", "TA", "Gd", "Ex"))
full$fence <- ordered(x = full$fence, levels = c("None", "MnWw", "GdWo", "MnPrv", "GdPrv"))

# distribution table for 'electrical' feature
table(full$electrical)

# which dataset contains the 'Mix' value
full %>% 
  filter(electrical == "Mix") %>% 
  select(df_id, id, electrical)

# remove the observation n°399
# note concerning filtering NA values in 'electrical' : https://github.com/tidyverse/dplyr/issues/3196
full <- full %>% 
  filter(is.na(electrical) | electrical != "Mix")

# relevel the 'electrical' factor
full$electrical <- ordered(x = full$electrical, levels = c("FuseP", "FuseF", "FuseA", "SBrkr"))

full %>% 
  mutate(built_after_1946 = year_built >= 1946) %>% 
  filter(ms_sub_class %in% c("20", "30", "60", "70", "120", "160")) %>% 
  select(ms_sub_class, built_after_1946) %>% 
  table()

# Export the 'full' dataset into a R object
saveRDS(object = full, file = "01-full_train_test.rds")