Introduction

The goal of this report is to provide a brief summary of the plans for creating the prediction algorithm and Shiny app. Potential words are predicted by frequencies of words and word pairs using a subsample with 1 percent of observations from the whole data. First, explore the training data and conduct data analysis. Then, design and train the first prediction model with the help of machine learning algorithms. Last, evaluate the first model and iterate the training steps to obtain an “optimal” model that performs the best according to the evaluation metrics.

Set Working Directory

setwd("C:/Users/Heidi_Tang/Desktop")

Load Libraries

library(ngram)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(knitr)
library(stringi)
library(tm)
## Loading required package: NLP
library(NLP)
library(ggplot2)
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
## 
##     annotate

Load and Read Training Data

destfile = "./Coursera-SwiftKey.zip"
if(!file.exists(destfile)){
  url = "https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip"
  file <- basename(url)
  download.file(url, file, method="curl")
  unzip(file)
}

blogs <- readLines("final/en_US/en_US.blogs.txt", encoding = 'UTF-8',warn = FALSE)
news <- readLines("final/en_US/en_US.news.txt", encoding = 'UTF-8',warn = FALSE)
twitter <- readLines("final/en_US/en_US.twitter.txt", encoding = 'UTF-8',warn = FALSE)

Summaries of Word and Line Counts

line_blogs<-length(blogs)
line_news<-length(news)
line_twitter<-length(twitter)

wc_blogs<-wordcount(blogs)
wc_news<-wordcount(news)
wc_twitter<-wordcount(twitter)

a<-rbind(line_blogs, line_news,line_twitter)
b<-rbind(wc_blogs, wc_news,wc_twitter)
c<-as.data.frame(cbind(a,b))
names(c)<-c("nr of lines","nr of words")
rownames(c)<-c("blogs", "news","twitter")
c
##         nr of lines nr of words
## blogs        899288    37334131
## news          77259     2643969
## twitter     2360148    30373543

Exploratory Analysis of Training Data

Randomly Select 1% of the Observations as a Subsample from the Data.

Sys.setenv(JAVA_HOME = 'C:/Program Files (x86)/Java/jre6')
library(rJava)
## java.home option:
## JAVA_HOME environment variable: C:/Program Files (x86)/Java/jre6
## Warning in fun(libname, pkgname): Java home setting is INVALID, it will be ignored.
## Please do NOT set it unless you want to override system settings.
library(RWeka)

set.seed(11000)
c_blogs <- sample(blogs, length(blogs)*0.01)
c_news <- sample(news, length(news)*0.01)
c_twitter <- sample(twitter, length(twitter)*0.01)
c_combi=c(c_blogs,c_news,c_twitter)

Basic Data Table

unigram_combi <- NGramTokenizer(c_combi, Weka_control(min = 1, max = 1))
bigram_combi <- NGramTokenizer(c_combi, Weka_control(min = 2, max = 2)) 
trigram_combi <- NGramTokenizer(c_combi, Weka_control(min = 3, max = 3)) 

unigram_combi<-data.frame(table(unigram_combi))%>%arrange(desc(Freq))
bigram_combi<-data.frame(table(bigram_combi))%>%arrange(desc(Freq))
trigram_combi<-data.frame(table(trigram_combi))%>%arrange(desc(Freq))

df_ngram<-as.data.frame(cbind(unigram_combi[1:12,],bigram_combi[1:12,],trigram_combi[1:12,]))
names(df_ngram)[c(2,4,6)]<-c("Freq1","Freq2","Freq3")
df_ngram
##    unigram_combi Freq1 bigram_combi Freq2  trigram_combi Freq3
## 1            the 26535       of the  2526        I don t   358
## 2             to 18621       in the  2364        I can t   211
## 3              I 16354          I m  1547       a lot of   181
## 4              a 14869      for the  1376 Thanks for the   180
## 5            and 14792       to the  1327     one of the   168
## 6             of 12985       on the  1214        I m not   159
## 7             in  9583        to be  1152        to be a   148
## 8            you  7957        don t   872    going to be   124
## 9             is  7906       at the   860      I want to   123
## 10           for  7372      and the   736     be able to   121
## 11          that  7121       I have   725     don t know   107
## 12            it  6911         is a   723       I have a   106

Plots: Rankings of Word(s) According to Frequency

library(ggplot2)
ggplot(df_ngram, aes(x=reorder(unigram_combi,Freq1), y=(Freq1))) +
  geom_bar(stat="Identity", fill="lightblue")+
  xlab("Unigrams") + ylab("Frequency")+
  ggtitle("Most common 12 Unigrams")+
  theme(axis.text.x=element_text(angle=90, hjust=1))

ggplot(df_ngram, aes(x=reorder(bigram_combi,Freq2), y=(Freq2))) +
  geom_bar(stat="Identity", fill="blue")+
  xlab("Bigrams") + ylab("Frequency")+
  ggtitle("Most common 12 Bigrams")+
  theme(axis.text.x=element_text(angle=90, hjust=1))

ggplot(df_ngram, aes(x=reorder(trigram_combi,Freq3), y=(Freq3))) +
  geom_bar(stat="Identity", fill="darkblue")+
  xlab("Trigrams") + ylab("Frequency")+
  ggtitle("Most common 12 Trigrams")+
  theme(axis.text.x=element_text(angle=90, hjust=1))

Conclusions

This report briefly illustrates how to create the prediction algorithm and the Shiny app. Machine learning algorithms are a helpful tool to search for “a better model” than the previous ones through iterating the training steps. We could better predict words from frequencies of word pairs.