Goal

to display that you’ve gotten used to working with the data and that you are on track to create your prediction algorithm. Please submit a report on R Pubs ( http://rpubs.com/ ) that explains your exploratory analysis and your goals for the eventual app and algorithm. This document should be concise and explain only the major features of the data you have identified and briefly summarize your plans for creating the prediction algorithm and Shiny app in a way that would be understandable to a non-data scientist manager. You should make use of tables and plots to illustrate important summaries of the data set. The motivation for this project is to: 1. Demonstrate that you’ve downloaded the data and have successfully loaded it in.2. Create a basic report of summary statistics about the data sets.3. Report any interesting findings that you amassed so far.4. Get feedback on your plans for creating a prediction algorithm and Shiny app.

Load the Required Libraries and Set the Working Environment

library(tm)
## Loading required package: NLP
library(RWeka)
library(stringi)
library(dplyr)
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
library(pryr)
## 
## Attaching package: 'pryr'
## The following object is masked from 'package:dplyr':
## 
##     where
## The following object is masked from 'package:tm':
## 
##     inspect
library(RColorBrewer)
library(ggplot2)
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
## 
##     annotate
library(wordcloud)
library(knitr)

setwd("/Users/skim/Desktop/temp/CS/Final")

Load data

Preliminary exploratory analysis:

files = c(blogs_addrs,news_addrs,twtr_addrs)
datasizes = sapply(files, function(x) {file.size(x)/1024^2})
WordCount=sapply(list(data_blogs, data_news, data_twtr), stri_stats_latex)[4,]
invisible(gc())
stats = rbind(datasizes,WordCount)
stats = as.data.frame(stats)
names(stats) = c('usblogs','usnews','ustwitter')
row.names(stats) = c('filesize(MB)','word_count')
kable(stats)
usblogs usnews ustwitter
filesize(MB) 2.004242e+02 1.962775e+02 1.593641e+02
word_count 3.757084e+07 3.449454e+07 3.045117e+07

Sampling and Cleaning the Data since the data is too large, I only need 1% of it to reduce computation power

set.seed(12345)
test_data <- c(sample(data_blogs, length(data_blogs) * 0.01),
              sample(data_news, length(data_news) * 0.01),
              sample(data_twtr, length(data_twtr) * 0.01) )

then we build the corpus and do some cleaning

# make a volatile corpus
testdata <- iconv(test_data, "UTF-8", "ASCII", sub="")
raw_corpus <- VCorpus(VectorSource(testdata))

clean_corpus <- function(corpus){
  corpus <- tm_map(corpus, content_transformer(tolower)) # convert to lower case
  corpus <- tm_map(corpus, stripWhitespace) # remove white space
  corpus <- tm_map(corpus, removePunctuation) # remove punctuation
  corpus <- tm_map(corpus,content_transformer(function(x) gsub("[[:digit:]]","",x)))# remove numbers
  corpus <- tm_map(corpus,content_transformer(function(x) gsub(" th", "",x))) # remove th (like 4th)
  corpus <- tm_map(corpus,content_transformer(function(x) gsub("http[[:alnum:]]*","",x))) # remove url
  corpus <- tm_map(corpus,content_transformer(function(x) iconv(x, "latin1", "ASCII", sub=""))) # remove non-ASCII characters
  corpus <- tm_map(corpus,content_transformer(function(x) gsub("([[:alpha:]])\\1{2,}", "\\1\\1", x))) # remove repeated alphabets in a word
  gc()
  return(corpus)
}

corpus <- clean_corpus(raw_corpus)

save(corpus,file='corpus.RData')

Preliminary predictive model

Build N-Gram model We have cleaned and sampled our data. we have done some preprocessing for our data. Now we can build our basic unigram, bi-grams and tri-grams. We are using RWeka packge for this purpose.

unigram <- function(x) NGramTokenizer(x, Weka_control(min=1, max=1))
bigram <- function(x) NGramTokenizer(x, Weka_control(min=2, max=2))
trigram <- function(x) NGramTokenizer(x, Weka_control(min=3, max=3))

unidtf <- TermDocumentMatrix(corpus, control=list(tokenize=unigram))
bidtf <- TermDocumentMatrix(corpus, control=list(tokenize=bigram))
tridtf <- TermDocumentMatrix(corpus, control=list(tokenize=trigram))

uni_tf <- findFreqTerms(unidtf, lowfreq = 50 )
bi_tf <- findFreqTerms(bidtf, lowfreq = 50 )
tri_tf <- findFreqTerms(tridtf, lowfreq = 10 )

uni_freq <- rowSums(as.matrix(unidtf[uni_tf, ]))
uni_freq <- data.frame(words=names(uni_freq), frequency=uni_freq)

bi_freq <- rowSums(as.matrix(bidtf[bi_tf, ]))
bi_freq <- data.frame(words=names(bi_freq), frequency=bi_freq)

tri_freq <- rowSums(as.matrix(tridtf[tri_tf, ]))
tri_freq <- data.frame(words=names(tri_freq), frequency=tri_freq)

head(tri_freq)

Summarising and plotting prelimiary analysis

Plotting N-grams data and wordCloud

wordcloud(words=uni_freq$words, freq=uni_freq$frequency, max.words=100, colors = brewer.pal(8, "Dark2"))

Uni_freq <- ggplot(data = uni_freq[order(-uni_freq$frequency),][1:15, ], aes(x = reorder(words, -frequency), y=frequency)) +
              geom_bar(stat="identity", fill="blue") + 
              ggtitle("Top Unigram") + xlab("words") +  ylab("frequency")

Uni_freq

Bi_freq <- ggplot(data = bi_freq[order(-bi_freq$frequency),][1:15, ], aes(x = reorder(words, -frequency), y=frequency)) +
  geom_bar(stat="identity", fill="red") + theme(axis.text.x = element_text(angle = 45)) + 
  ggtitle("Top Bigram") + xlab("words") +  ylab("frequency")
  
Bi_freq

Tri_freq <- ggplot(data = tri_freq[order(-tri_freq$frequency),][1:15, ], aes(x = reorder(words, -frequency), y=frequency)) +
  geom_bar(stat="identity", fill="cyan") + theme(axis.text.x = element_text(angle = 45)) + 
  ggtitle("Top Trigram") + xlab("words") +  ylab("frequency")

Tri_freq