library(RCurl)
library(xml2)
library(rvest)
library(knitr)
library(tidyr)
library(dplyr)
library(readr)
library(mongolite)
library(dummies)
library(ggplot2)

# Function to generate URLS
itter <- c("", "&start=10", "&start=20", "&start=30")
IndeedCities <- function(jobTitle, city, state){
  url <- c() #To make a list with multiple returns.
  for(i in itter){ # create urls
  i <- paste0("https://www.indeed.com/jobs?q=",
                gsub(" ", replacement = "+", jobTitle),
                "&l=",
                gsub(" ", replacement = "+", city),
                "%2C+", 
                state, 
                i,
                sep = "")
  url <- c(url, i) #Update list of urls
  }
  return(url)
}

# This is easier than updating a datafram from a function and using a loop. 
nyc <-  IndeedCities("Data Scientist", "New York", "NY")
bost <- IndeedCities("Data Scientist", "Boston", "MA")
sanFran <- IndeedCities("Data Scientist", "San Francisco", "CA")
hust <- IndeedCities("Data Scientist", "Houston", "TX")
sea <- IndeedCities("Data Scientist", "Seattle", "WA")

df1 <- data.frame(CompanyName = as.character(),
                  TitleName = as.character(),
                  City = as.character(),
                  ExperienceList = as.character())
IndeedData <- function(x){
  for(i in c(1:4)){
    ineeedPage <- read_html(x[i]) # reading to make a list.

   #Creating columns of the dataframe:

   ExperienceList <- ineeedPage %>%
      html_nodes(".experienceList") %>%
      html_text()

    TitleName <- ineeedPage %>%
     html_nodes(".jobtitle") %>%
     html_text()

    CompanyName <- ineeedPage %>%
      html_nodes(".company") %>%
      html_text()

    City <- ineeedPage %>%
      html_nodes(".location") %>%
      html_text()
    # Make a list of all the text in the webpage. 
    test <- ineeedPage %>%
      html_nodes(".result") %>%
      html_text()
# Make sure that experience is actually included.
    for(j in 1:length(test)) {
      if (grepl("Desired Experience:",x = test[j])) { # Double checking to make sure we don't skip NAs
      } else {
        ExperienceList <-  append(ExperienceList, NA, after = (j))
  }
}
    df <- data.frame(CompanyName,TitleName, City, ExperienceList)
    df1 <- rbind(df1,df)
  }
 return(df1)
}

nycDF <- IndeedData(nyc) 
bostDF <- IndeedData(bost)
sanFranDF <- IndeedData(sanFran)
hustDF <- IndeedData(hust)
seaDF <- IndeedData(sea)

df <- rbind(nycDF, bostDF, sanFranDF, hustDF, seaDF)
df$City <- gsub("\\d.*","",df$City)
df$CompanyName <- gsub("\\n","",df$CompanyName)
df$TitleName <- gsub("\\n","",df$TitleName)
df$City <- trimws(df$City) # Removes white space
df$ExperienceList <- trimws(df$ExperienceList) # Removes white space on either end. 
df %>% head() %>% kable()

# Explained in the presentation rmd.
skillCount <- paste(df$ExperienceList, collapse = ",") 
skillCount <-  strsplit(skillCount, ",")[[1]]
skillCount <- gsub(" ", "", skillCount)
skillTable <-  table(skillCount) %>% as.data.frame()

ggplot(skillTable, aes(x = reorder(skillCount, Freq), y = Freq)) + 
  geom_bar(stat = "identity") +
  ggtitle("Percentage of Non- Missing Values") +
  coord_flip() 

# SEtting up MongoDB
m <- mongo(url = "mongodb://127.0.0.1:27017" ) # Database ports

jsonlite::stream_out(df, file("skillsDF.json"), verbose = FALSE)
mt <- mongo("skillsDF")
mt$import(file("skillsDF.json"))
mt$find() %>% head(30) %>% kable()