Adding ft_string_indexer

Initial Setup

library(sparklyr)
library(dplyr)
library(nycflights13)
sc <- spark_connect(master = "local", version = "2.1.0")
f <-  copy_to(sc, flights)

fs <- f %>%
  filter(!is.na(arr_delay)) %>%
  ft_string_indexer("origin", "origin_ca") %>%
  ft_string_indexer("dest", "dest_ca") %>%
  sdf_partition(train = 0.8, test = 0.2) 
  

With ft_string_indexer

with_cat <- system.time({
  fs_ml <- fs$train %>%
  select(origin_ca,
         dest_ca,
         arr_delay) %>%
    ml_random_forest(arr_delay~., max.bins = 200)
  
  fs_pr <- sdf_predict(fs_ml, fs$test)
  
  })

* No rows dropped by 'na.omit' call

with_cat

   user  system elapsed 
  0.145   0.017   6.053 

With out ft_string_indexer

wo_cat <- system.time({
  
  fs_ml <- fs$train %>%
  select(origin,
         dest,
         arr_delay) %>%
    ml_random_forest(arr_delay~., max.bins = 200)
  
 
  fs_pr <- sdf_predict(fs_ml, fs$test)
  
  
  
  
})

* No rows dropped by 'na.omit' call

wo_cat

   user  system elapsed 
  1.016   0.086  15.618 

spark_disconnect(sc)