Summary

This is the R Markdown document of milestone report for capstone project.

To finish this capstone project, I will start with downloading the text data from given link (https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip). The data includes blogs, news and twitters text files. They are available in multiple languages but I’ll only deal with the data files in English.Then I’ll analyze the data using text mining and NLP approaches to discover the structure in the data and how words are put together. After that, I’ll be building a predictive text model to predict the most likely next word. Finally, I’ll create a data product using shiny app and provide instruction slides documentation.

As required, in this milestone report, we’ll
1. Download and unzip data file Coursera-SwiftKey.zip.
2. Do some basic summaries to blog, news and twitter data files in en_US directory.
3. Do some basic plots to illustrate features of the data.

Load data

First, we’ll download dataset file and unzip it:

setwd("~/course/ds_coursera/capstone")

file.url <- "https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip"

if(!file.exists("./Coursera-SwiftKey.zip")) {
    download.file(file.url, destfile="./Coursera-SwiftKey.zip", method="curl")
}

if(!file.exists("./final")) {
    unzip("./Coursera-SwiftKey.zip")
}

Second, we’ll use readLines to read those files to R.

if(!file.exists("./blogs.RData")) {
    blogs.con <- file("./final/en_US/en_US.blogs.txt", "rb")
    blogs <- readLines(blogs.con, encoding="UTF-8", skipNul = TRUE, warn = FALSE) 
    close(blogs.con)
    save(blogs, file="blogs.RData")
}

if(!file.exists("./news.RData")) {
    news.con <- file("./final/en_US/en_US.news.txt", "rb")
    news <- readLines(news.con, encoding="UTF-8", skipNul = TRUE, warn = FALSE) 
    close(news.con)
    save(news, file="news.RData")
}

if(!file.exists("./twitter.RData")) {
    twitter.con <- file("./final/en_US/en_US.twitter.txt", "rb")
    twitter <- readLines(twitter.con, encoding="UTF-8", skipNul = TRUE, warn = FALSE) 
    close(twitter.con)
    save(twitter, file="twitter.RData")
}

Explortary

Data first glance: line count

load("blogs.RData")
load("news.RData")
load("twitter.RData")

# get total line numbers for each file
blogs.nrow <- length(blogs)
news.nrow <- length(news)
twitter.nrow <- length(twitter)

row.data <- c(blogs.nrow, news.nrow, twitter.nrow)
names(row.data) <- c("blogs", "news", "twitter")
row.data
##   blogs    news twitter 
##  899288 1010242 2360148
barplot(row.data, ylab="Number of Lines", main="Bar Plot to Number of Lines")

As the files are big, we do sampling to read random chosen lines from each files for further analysis

# set random number to do sampling - only sample 0.1% data to do analysis.
set.seed(1234)
blogs.random <- as.logical(rbinom(n=blogs.nrow, size=1, prob=0.001))
set.seed(2345)
news.random <- as.logical(rbinom(n=news.nrow, size=1, prob=0.001))
set.seed(3456)
twitter.random <- as.logical(rbinom(n=twitter.nrow, size=1, prob=0.001))

# do sampling and create subsets
blogs.sample <- blogs[blogs.random]
news.sample <- news[news.random]
twitter.sample <- twitter[twitter.random]

save(blogs.sample, file="blogs.sample.RData")
save(news.sample, file="news.sample.RData")
save(twitter.sample, file="twitter.sample.RData")

Further exploratory analysis 1 - number of words distribution

load("blogs.sample.RData")
load("news.sample.RData")
load("twitter.sample.RData")

# number of word distribution to each sample file
library(stringi)

blogs.sample.nword <- stri_count(blogs.sample,regex="\\S+")
news.sample.nword <- stri_count(news.sample,regex="\\S+")
twitter.sample.nword <- stri_count(twitter.sample,regex="\\S+")

library(ggplot2)

ggplot(data.frame(x=blogs.sample.nword), aes(x)) + geom_histogram(bins=30) +
    labs(title="Histogram of document word count from blogs sample data") +
    labs(x="Number of words", y="Document Count")

ggplot(data.frame(x=news.sample.nword), aes(x)) + geom_histogram(bins=30) +
    labs(title="Histogram of document word count from news sample data") +
    labs(x="Number of words", y="Document Count")

ggplot(data.frame(x=blogs.sample.nword), aes(x)) + geom_histogram(bins=30) +
    labs(title="Histogram of document word count from twitter sample data") +
    labs(x="Number of words", y="Document Count")

Further exploratory analysis 2 - corpus, tokenization and wordcloud

load("blogs.sample.RData")
load("news.sample.RData")
load("twitter.sample.RData")

full.sample <- c(blogs.sample, news.sample, twitter.sample)

#full.sample <- iconv(full.sample, from="UTF-8", to="latin1", sub=" ")
#full.sample <- tolower(full.sample)

library(tm)
## Loading required package: NLP
## 
## Attaching package: 'NLP'
## The following object is masked from 'package:ggplot2':
## 
##     annotate
full.sample.corpus <- VectorSource(full.sample)
full.sample.corpus <- VCorpus(full.sample.corpus)
full.sample.corpus <- tm_map(full.sample.corpus, content_transformer(tolower))
full.sample.corpus <- tm_map(full.sample.corpus, removeNumbers)
full.sample.corpus <- tm_map(full.sample.corpus, removePunctuation)
full.sample.corpus <- tm_map(full.sample.corpus, stripWhitespace)
full.sample.corpus <- tm_map(full.sample.corpus, removeWords, stopwords(kind="en"))

save(full.sample.corpus, file="full.sample.corpus.RData")
load("full.sample.corpus.RData")

library(RWeka)
library(wordcloud)
## Loading required package: RColorBrewer
full.sample.dtm1 <- DocumentTermMatrix(full.sample.corpus)
full.sample.dtm1.new <- removeSparseTerms(full.sample.dtm1,sparse = 0.99)
full.sample.dtm1.m <- as.matrix(full.sample.dtm1.new)
full.sample.frequency1 <- colSums(full.sample.dtm1.m)
wordcloud(names(full.sample.frequency1), full.sample.frequency1, min.freq = 15, 
          scale = c(4,.2), colors = brewer.pal(6, 'Dark2'))

#full.sample.frequency1.sort <- sort(full.sample.frequency1, decreasing = TRUE)
#head(full.sample.frequency1.sort,10)


ngram.tokenizer.2 <- function (x) NGramTokenizer(x, Weka_control(min=2, max=2))
full.sample.dtm2 <- DocumentTermMatrix(full.sample.corpus, control=list(tokenize=ngram.tokenizer.2))
full.sample.dtm2.new <- removeSparseTerms(full.sample.dtm2,sparse = 0.999)
full.sample.dtm2.m <- as.matrix(full.sample.dtm2.new)
full.sample.frequency2 <- colSums(full.sample.dtm2.m)
wordcloud(names(full.sample.frequency2), full.sample.frequency2, min.freq = 5, 
          scale = c(2,.2), colors = brewer.pal(6, 'Dark2'))

#full.sample.frequency2.sort <- sort(full.sample.frequency2, decreasing = TRUE)
#head(full.sample.frequency2.sort, 10)

save(full.sample.dtm1.m, file="full.sample.dtm1.m.RData")
save(full.sample.dtm2.m, file="full.sample.dtm2.m.RData")