Overview

The goal of this project is just to display that you’ve gotten used to working with the data and that you are on track to create your prediction algorithm. Please submit a report on R Pubs ( http://rpubs.com/ ) that explains your exploratory analysis and your goals for the eventual app and algorithm. This document should be concise and explain only the major features of the data you have identified and briefly summarize your plans for creating the prediction algorithm and Shiny app in a way that would be understandable to a non-data scientist manager. You should make use of tables and plots to illustrate important summaries of the data set. The motivation for this project is to: 1. Demonstrate that you’ve downloaded the data and have successfully loaded it in.2. Create a basic report of summary statistics about the data sets.3. Report any interesting findings that you amassed so far.4. Get feedback on your plans for creating a prediction algorithm and Shiny app.

Import Libraries

require(tm)
require(SnowballC)
require(wordcloud)
require(RWeka)

Read Data

twitter <- readLines("final/en_US/en_US.twitter.txt", encoding = "UTF-8", skipNul = TRUE)
blogs <- readLines("final/en_US/en_US.blogs.txt", encoding = "UTF-8", skipNul = TRUE)

## News file has special characters so it is better to read it as a binary file
bin.news <- file("final/en_US/en_US.news.txt", open="rb")
news <- readLines(bin.news, encoding="UTF-8")
close(bin.news)
rm(bin.news)

Sample Data

set.seed(5454568)
sampleTwitter <- twitter[sample(1:length(twitter),1000)]
sampleBlogs <- blogs[sample(1:length(blogs),1000)]
sampleNews <- news[sample(1:length(news),1000)]

## Combine data samples
sampleData <- c(sampleTwitter,sampleBlogs,sampleNews)

## Save sample data and remove data not needed to free memory
writeLines(sampleData, "sampleData.txt")
rm(twitter,news,blogs,sampleTwitter,sampleNews,sampleBlogs)

head(sampleData)
## [1] "haha I bet they're all great photos! :D And Kellan is cute. (;"                                                               
## [2] "next thing I know, you're gonna want to pass on Loe vs. LHH too."                                                             
## [3] "If, God forbid, Newt becomes nominee and then president, ya'll will find the Trotters at General Delivery, Vancouver, Canada."
## [4] "I actually thought it was a reasonable trade, for what it's worth. (I even thought it made the Lakers slightly *worse*)"      
## [5] "\": Don't give up I know you can see All the world and the mess that were making Can't give up\""                             
## [6] "As an internal change communication strategist that is likely a beneficial read for me!"

CLean Data

sampleData <- readLines("sampleData.txt", encoding="UTF-8")
corpus <- VCorpus(VectorSource(sampleData))

## Remove space, punctuation, numbers, whitespace, stopwords and change to lowercase
toSpace <- content_transformer(function(x, pattern) gsub(pattern, " ", x))
corpus <- tm_map(corpus, toSpace, "\"|/|@|\\|")
corpus <- tm_map(corpus, toSpace, "[^[:graph:]]")
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, stripWhitespace)
corpus <- tm_map(corpus, stemDocument)
corpus <- tm_map(corpus, PlainTextDocument)
corpus <- tm_map(corpus, removeWords, stopwords("english"))

Generating Word Cloud

wordcloud(corpus, min.freq=5, max.words=101, random.order=TRUE,
          rot.per=0.5, colors=brewer.pal(8, "Set2"), use.r.layout=FALSE)

Findings

corpus.dataframe <- data.frame(text = unlist(sapply(corpus, '[', 'content')), stringsAsFactors = F)

uniGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 1, max = 1))))
biGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 2, max = 2))))
triGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 3, max = 3))))

#order by decreasing frequency
unigram <- uniGramToken[order(uniGramToken$Freq, decreasing = TRUE),]
bigram  <- biGramToken[order(biGramToken$Freq, decreasing = TRUE),]
trigram <- triGramToken[order(triGramToken$Freq, decreasing = TRUE),]

Unigrams

par(mfrow = c(1, 1))
par(mar=c(5,4,2,0))
barplot(unigram[1:30,2], 
        names.arg=unigram[1:30,1], 
        col = "red", 
        main="Most commonly used Words", 
        las=2, 
        ylab = "Frequency")

head(unigram)
##       Var1 Freq
## 6423   one  276
## 7828  said  272
## 10054 will  267
## 3795   get  229
## 5304  like  226
## 1396   can  209

Bigram

par(mar=c(8.5,4,2,1))
barplot(bigram[1:30,2], 
        names.arg=bigram[1:30,1], 
        col = "blue", 
        main="Most commonly used two word combinations", 
        las=2, 
        ylab = "Frequency")

head(bigram)
##               Var1 Freq
## 21396    last year   24
## 18039  high school   18
## 26296     new york   18
## 32894    right now   17
## 11056    dont know   15
## 22819 look forward   13

Trigram

par(mar=c(8.5,4,2,1))
barplot(trigram[1:30,2], 
        names.arg=trigram[1:30,1], 
        col = "green", 
        main="Most commonly used three word combinations", 
        las=2, 
        ylab = "Frequency")

head(trigram)
##                         Var1 Freq
## 20704 internet servic provid    4
## 7415           cinco de mayo    3
## 17472      gov chris christi    3
## 31365         pleas let know    3
## 33477    realli look forward    3
## 35926        say take villag    3

Plans for Future

I am planning to use ngram to calculate the probabilities of the next word occuring with respect to previous words. For the Shiny app, the plan is to create an app with an UI where the user can enter a string and then the model will then give a suggested word to update the next word.