Coursera Data Science Capstone: Milestone Report

Introduction

In this capstone we will be applying data science in the area of natural language processing. We’ll describe the major features of the training data with our exploratory data analysis and present how to create the predictive model.

Data download

Our first step is to download the dataset from the given link.

## Init variables
capstonedatasetlink <- "https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip"
projectfolderpath <- "./projectData"
zipname <- "SwiftKey.zip"
zippath <- paste0(projectfolderpath, "/", zipname)
datasetfolder <- "final"
datasetpath<- paste0(projectfolderpath, "/", datasetfolder) 

## Check if directory already exists?
if(!file.exists(projectfolderpath)){
  dir.create(projectfolderpath)
}
## Check if zip has already been downloaded in projectData directory?
if(!file.exists(zippath)){
  download.file(capstonedatasetlink,destfile=zippath,mode = "wb")
}
## Check if zip has already been unzipped?
if(!file.exists(datasetfolder)){
  unzip(zipfile=zippath,exdir=projectfolderpath)
}

Data load

After downloading the datset, we are starting to read the dataset that we need. The dataset consists of text files from 3 different sources: news, blogs and twitter. In this project, we will focus on the Englis US datasets.

enpath <- file.path(datasetpath , "en_US")
files<-list.files(enpath, recursive=TRUE)

# Lets make a file connection of the twitter data set
con <- file(paste0(enpath, "/en_US.twitter.txt"), "r") 
#lineTwitter<-readLines(con, 100, skipNul = TRUE)
lineTwitter<-readLines(con, skipNul = TRUE)
# Close the connection handle when you are done
close(con)

# Lets make a file connection of the blog data set
con <- file(paste0(enpath, "/en_US.blogs.txt"), "r") 
#llineBlogs<-readLines(con, 100, skipNul = TRUE)
lineBlogs<-readLines(con, skipNul = TRUE)
# Close the connection handle when you are done
close(con)

# Lets make a file connection of the news data set
con <- file(paste0(enpath, "/en_US.news.txt"), "r") 
#lineNews<-readLines(con, 100, skipNul = TRUE)
lineNews<-readLines(con, skipNul = TRUE)
# Close the connection handle when you are done
close(con)

Data summary

A summary for the US blogs, news and twitter files can be seen on the table below.

# Get file sizes
lineBlogsSize <- file.info(paste0(enpath, "/en_US.blogs.txt"))$size / 1024 ^ 2
lineNewsSize <- file.info(paste0(enpath, "/en_US.news.txt"))$size / 1024 ^ 2
lineTwitterSize <- file.info(paste0(enpath, "/en_US.twitter.txt"))$size / 1024 ^ 2
# Get words in files
lineBlogsWords <- stri_count_words(lineBlogs)
lineNewsWords <- stri_count_words(lineNews)
lineTwitterWords <- stri_count_words(lineTwitter)
# Summary of the data sets
data.frame(source = c("blogs", "news", "twitter"),
           fileSize_MB = c(lineBlogsSize, lineNewsSize, lineTwitterSize),
           wordCounts = c(sum(lineBlogsWords), sum(lineNewsWords), sum(lineTwitterWords)),
           wordMeans = c(mean(lineBlogsWords), mean(lineNewsWords), mean(lineTwitterWords)),
           lineCounts = c(length(lineBlogs), length(lineNews), length(lineTwitter)))

Data sampling

We observe that the files have a considerable amount of data (e.g. 2 million lines for twitter file). In order to create a model, we will need to take a sample of the data to avoid using to much memory.

percent <- 0.01
sampling <- function(data, percent){
  return(data[as.logical(rbinom(length(data),1,percent))])
}
  
sampleBlogs   <- sampling(lineBlogs, percent)
sampleNews   <- sampling(lineNews, percent)
sampleTwitter   <- sampling(lineTwitter, percent)

remove(lineBlogs)
remove(lineNews)
remove(lineTwitter)


# Get words in files
lineBlogsWords <- stri_count_words(sampleBlogs)
lineNewsWords <- stri_count_words(sampleNews)
lineTwitterWords <- stri_count_words(sampleTwitter)
# Summary of the data sets
data.frame(source = c("blogs", "news", "twitter"),
           wordCounts = c(sum(lineBlogsWords), sum(lineNewsWords), sum(lineTwitterWords)),
           wordMeans = c(mean(lineBlogsWords), mean(lineNewsWords), mean(lineTwitterWords)),
           lineCounts = c(length(sampleBlogs), length(sampleNews), length(sampleTwitter)))

NA

Cleaning and Tokenization

In order to be able manipulate our data, we will create a cleaned corpus, which will consist of the three sample text files. This involves removing punctuations, numbers, excess whitespace, stopwords, and changing the text to lower case.

allSamples <- c(sampleBlogs, sampleNews, sampleTwitter)
corpus <- VCorpus(VectorSource(allSamples))
#clean
corpus <- tm_map(corpus, tolower)
corpus <- tm_map(corpus, stripWhitespace) 
corpus <- tm_map(corpus, removeNumbers) 
corpus <- tm_map(corpus, removePunctuation, preserve_intra_word_dashes=TRUE)
corpus <- tm_map(corpus, removeWords, stopwords("en"))

#tokenize
unigram <- NGramTokenizer(corpus, Weka_control(min = 1, max = 1,delimiters = " \\r\\n\\t.,;:\"()?!"))
biGram <- NGramTokenizer(corpus, Weka_control(min = 2, max = 2,delimiters = " \\r\\n\\t.,;:\"()?!"))
triGram <- NGramTokenizer(corpus, Weka_control(min = 3, max = 3,delimiters = " \\r\\n\\t.,;:\"()?!"))

## explore and plot
# converting tokens of n-grams into tables
uniGramTable <- data.frame(table(unigram))
biGramTable <- data.frame(table(biGram))
triGramTable <- data.frame(table(triGram))

  
uniGramTable <- uniGramTable[order(uniGramTable$Freq,decreasing = TRUE),]
biGramTable <- biGramTable[order(biGramTable$Freq,decreasing = TRUE),]
triGramTable <- triGramTable[order(triGramTable$Freq,decreasing = TRUE),]

top1gram <- uniGramTable[1:10,]
colnames(top1gram) <- c("word","count")
top2gram <- biGramTable[1:10,]
colnames(top2gram) <- c("word","count")
top3gram <- triGramTable[1:10,]
colnames(top3gram) <- c("word","count")

Next steps

We will create a prediction algorithm and a Shiny app in a way that would be understandable to a non-data scientist manager. We will be using the n-gram model with to compute the probability of the next word occuring. The input will be tokenized and the last words will be isolated to get the hightest probability next word. Our Shiny app will be interactive and any non-data scientist users will be able to easily navigate through our it.