#Inputs to loop
base_url <- "https://www.trulia.com/for_sale/Chicago,IL/SINGLE-FAMILY_HOME_type/"
pages <- 150
trulia_file <- "Trulia_file.csv"
aggregate_df <- data.frame()
reg_ex1 <- "var appState = "
reg_ex2 <- ";\\n  var googleMapURL ="
my_samp <- seq(1, 3, by = .01)

#Loop to scrape pages
if (!trulia_file %in% list.files(getwd())){
  for (i in 1:pages){
    #pagination
    current_url <- ifelse(i == 1,
                          base_url,
                          paste0(base_url, i, "_p/")
                          )
    #get html    
    trulia_html <- current_url %>%
      read_html() %>%
      html_nodes("script") %>%
      html_text()
    
    #get json from html
    json_text <- trulia_html[str_detect(trulia_html, reg_ex1)]
    begin <- as.integer(str_locate(json_text, reg_ex1)[1, 2])
    ending <- as.integer(str_locate(json_text, reg_ex2)[1, 1]) - 1
    
    #parse the JSON
    json <- json_text %>%
      str_sub(begin, ending) %>%
      str_trim() %>%
      fromJSON()
  
    #store data in DF  
    current_df <- data.frame(iteration = i,
                            id = json$page$cards$id,
                            price = json$page$cards$price,
                            zip_code = json$page$cards$zip,
                            location = json$page$cards$footer$location)
  
    aggregate_df <- rbind(aggregate_df, current_df)
  
    #delay
    rand_delay <- sample(my_samp, 1, replace = T)
    Sys.sleep(rand_delay)
  }
  write.csv(aggregate_df, file = trulia_file)
}  

trulia_data <- read.csv(trulia_file, stringsAsFactors = F)

trulia_df <- trulia_data %>% transmute(sell_price = as.integer(str_replace_all(price, 
    "\\$|\\+|,", "")), zip_code = as.character(zip_code)) %>% na.omit() %>% 
    group_by(zip_code) %>% summarise(median_sell_price = median(sell_price), 
    n = n())

glimpse(trulia_df)

table(divvy_data$postalCode == "")

FALSE  TRUE 
  111   470 

coordinates <- cbind(divvy_data$longitude, divvy_data$latitude)
divvy_file <- "DivvyAddresses.csv"

if (!divvy_file %in% list.files(getwd())) {
    ## Code citation: http://stackoverflow.com/a/22919546
    address <- do.call(rbind, lapply(1:nrow(coordinates), function(i) revgeocode(coordinates[i, 
        ])))
    write.csv(data.frame(address = address), file = divvy_file)
}


divvy <- cbind(divvy_data, read.csv(divvy_file))