The goal of this project is just to display that you’ve gotten used to working with the data and that you are on track to create your prediction algorithm. Please submit a report on R Pubs ( http://rpubs.com/ ) that explains your exploratory analysis and your goals for the eventual app and algorithm. This document should be concise and explain only the major features of the data you have identified and briefly summarize your plans for creating the prediction algorithm and Shiny app in a way that would be understandable to a non-data scientist manager. You should make use of tables and plots to illustrate important summaries of the data set. The motivation for this project is to: 1. Demonstrate that you’ve downloaded the data and have successfully loaded it in.2. Create a basic report of summary statistics about the data sets.3. Report any interesting findings that you amassed so far.4. Get feedback on your plans for creating a prediction algorithm and Shiny app.
require(tm)
require(SnowballC)
require(wordcloud)
require(RWeka)
twitter <- readLines("final/en_US/en_US.twitter.txt", encoding = "UTF-8", skipNul = TRUE)
blogs <- readLines("final/en_US/en_US.blogs.txt", encoding = "UTF-8", skipNul = TRUE)
## News file has special characters so it is better to read it as a binary file
bin.news <- file("final/en_US/en_US.news.txt", open="rb")
news <- readLines(bin.news, encoding="UTF-8")
close(bin.news)
rm(bin.news)
set.seed(5454568)
sampleTwitter <- twitter[sample(1:length(twitter),1000)]
sampleBlogs <- blogs[sample(1:length(blogs),1000)]
sampleNews <- news[sample(1:length(news),1000)]
## Combine data samples
sampleData <- c(sampleTwitter,sampleBlogs,sampleNews)
## Save sample data and remove data not needed to free memory
writeLines(sampleData, "sampleData.txt")
rm(twitter,news,blogs,sampleTwitter,sampleNews,sampleBlogs)
head(sampleData)
## [1] "haha I bet they're all great photos! :D And Kellan is cute. (;"
## [2] "next thing I know, you're gonna want to pass on Loe vs. LHH too."
## [3] "If, God forbid, Newt becomes nominee and then president, ya'll will find the Trotters at General Delivery, Vancouver, Canada."
## [4] "I actually thought it was a reasonable trade, for what it's worth. (I even thought it made the Lakers slightly *worse*)"
## [5] "\": Don't give up I know you can see All the world and the mess that were making Can't give up\""
## [6] "As an internal change communication strategist that is likely a beneficial read for me!"
sampleData <- readLines("sampleData.txt", encoding="UTF-8")
corpus <- VCorpus(VectorSource(sampleData))
## Remove space, punctuation, numbers, whitespace, stopwords and change to lowercase
toSpace <- content_transformer(function(x, pattern) gsub(pattern, " ", x))
corpus <- tm_map(corpus, toSpace, "\"|/|@|\\|")
corpus <- tm_map(corpus, toSpace, "[^[:graph:]]")
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers)
corpus <- tm_map(corpus, stripWhitespace)
corpus <- tm_map(corpus, stemDocument)
corpus <- tm_map(corpus, PlainTextDocument)
corpus <- tm_map(corpus, removeWords, stopwords("english"))
wordcloud(corpus, min.freq=5, max.words=101, random.order=TRUE,
rot.per=0.5, colors=brewer.pal(8, "Set2"), use.r.layout=FALSE)
corpus.dataframe <- data.frame(text = unlist(sapply(corpus, '[', 'content')), stringsAsFactors = F)
uniGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 1, max = 1))))
biGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 2, max = 2))))
triGramToken <- data.frame(table(NGramTokenizer(corpus.dataframe, Weka_control(min = 3, max = 3))))
#order by decreasing frequency
unigram <- uniGramToken[order(uniGramToken$Freq, decreasing = TRUE),]
bigram <- biGramToken[order(biGramToken$Freq, decreasing = TRUE),]
trigram <- triGramToken[order(triGramToken$Freq, decreasing = TRUE),]
par(mfrow = c(1, 1))
par(mar=c(5,4,2,0))
barplot(unigram[1:30,2],
names.arg=unigram[1:30,1],
col = "red",
main="Most commonly used Words",
las=2,
ylab = "Frequency")
head(unigram)
## Var1 Freq
## 6423 one 276
## 7828 said 272
## 10054 will 267
## 3795 get 229
## 5304 like 226
## 1396 can 209
par(mar=c(8.5,4,2,1))
barplot(bigram[1:30,2],
names.arg=bigram[1:30,1],
col = "blue",
main="Most commonly used two word combinations",
las=2,
ylab = "Frequency")
head(bigram)
## Var1 Freq
## 21396 last year 24
## 18039 high school 18
## 26296 new york 18
## 32894 right now 17
## 11056 dont know 15
## 22819 look forward 13
par(mar=c(8.5,4,2,1))
barplot(trigram[1:30,2],
names.arg=trigram[1:30,1],
col = "green",
main="Most commonly used three word combinations",
las=2,
ylab = "Frequency")
head(trigram)
## Var1 Freq
## 20704 internet servic provid 4
## 7415 cinco de mayo 3
## 17472 gov chris christi 3
## 31365 pleas let know 3
## 33477 realli look forward 3
## 35926 say take villag 3
I am planning to use ngram to calculate the probabilities of the next word occuring with respect to previous words. For the Shiny app, the plan is to create an app with an UI where the user can enter a string and then the model will then give a suggested word to update the next word.