Load data

First of all, we load the data, the packages needed, set the working directory, and read the raw data in each of the three groups; blogs, twitter and news.

library(stringi)
library(ggplot2)
library(gt)
library(tm)
## Loading required package: NLP
## 
## Attaching package: 'NLP'
## The following object is masked from 'package:ggplot2':
## 
##     annotate
library(NLP)
library(RWeka)
library(plotly)
## 
## Attaching package: 'plotly'
## The following object is masked from 'package:ggplot2':
## 
##     last_plot
## The following object is masked from 'package:stats':
## 
##     filter
## The following object is masked from 'package:graphics':
## 
##     layout
setwd("C:/Users/andre/OneDrive - unizar.es/Data Science/RStudio/Data products/DataScience_Coursera_Repo/Final Capstone")

blogs<-readLines("en_US.blogs.txt",warn=FALSE,encoding="UTF-8")
news<-readLines("en_US.news.txt",warn=FALSE,encoding="UTF-8")
twitter<-readLines("en_US.twitter.txt",warn=FALSE,encoding="UTF-8")

Getting the summarised parameters

Using base functions we can get the size of each file, as well as the string lengths (the text length) and the number of characters.

size_blogs<-file.size(path="en_US.blogs.txt")/2^20
size_news<-file.size(path="en_US.news.txt")/2^20
size_twitter<-file.size(path="en_US.twitter.txt")/2^20

len_blogs<-length(blogs)
len_news<-length(news)
len_twitter<-length(twitter)

numberchar_blog<-sum(nchar(blogs))
numberchar_news<-sum(nchar(news))
numberchar_twitter<-sum(nchar(twitter))

nword_blogs<-stri_stats_latex(blogs)[4]
nword_news<-stri_stats_latex(news)[4]
nword_twitter<-stri_stats_latex(twitter)[4]

table<-data.frame("File Name"=c("Blogs","News","Twitter"),
                  "File Size(MB)"=c(size_blogs,size_news,size_twitter),
                  "Num of rows"=c(len_blogs,len_news,len_twitter),
                  "Num of character"=c(numberchar_blog,numberchar_news,numberchar_twitter),
                  "Num of words"=c(nword_blogs,nword_news,nword_twitter))

 cols_label(gt(table),
    File.Name = "File Name",
    File.Size.MB. = "Size in MB",
    Num.of.rows  = "Number of rows",
    Num.of.character="Number of characters",
    Num.of.words="Number of Words")
File Name Size in MB Number of rows Number of characters Number of Words
Blogs 200.4242 899288 206824505 37570839
News 196.2775 77259 15639408 2651432
Twitter 159.3641 2360148 162096031 30451128

Cleaning the data

First, the seed is set and, after every conversion, the former variables will be removed for memory efficiency reasons. I am using samples of 5% the original data set.

set.seed(2002)

blog_converted<-iconv(blogs,"latin1","ASCII",sub="")
news_converted<-iconv(news,"latin1","ASCII",sub="")
twitter_converted<-iconv(twitter,"latin1","ASCII",sub="")

rm(blogs)
rm(news)
rm(twitter)

# sample data set only 1% of each file
sample_data<-c(sample(blog_converted,length(blog_converted)*0.05),
               sample(news_converted,length(news_converted)*0.05),
               sample(twitter_converted,length(twitter_converted)*0.05))

rm(blog_converted)
rm(news_converted)
rm(twitter_converted)

Build corpus

Then, I build the corpus, set the mapping, classify the characters and make a data frame with all that summarized data.

corpus<-VCorpus(VectorSource(sample_data))
corpus1<-tm_map(corpus,removePunctuation)
corpus2<-tm_map(corpus1,stripWhitespace)
corpus3<-tm_map(corpus2,tolower)
corpus4<-tm_map(corpus3,removeNumbers)
corpus5<-tm_map(corpus4,PlainTextDocument)
corpus6<-tm_map(corpus5,removeWords,stopwords("english"))

corpus_result<-data.frame(text=unlist(sapply(corpus6,'[',"content")),stringsAsFactors = FALSE)
gt(head(corpus_result))
text
came music playing im sure went wave stage nearby
now refusing cook kitchen thrown lot food away will eat items fridge time nervously enter kitchen mentally prepare make sure im holding anything breakable shout loudly clap give chance least run away hide dont see strolling around kitchen surfaces
english restoration faction british army removed richard cromwell lord protector commonwealth
sacha baron cohen back new character dictator heroic story dictator risks life ensure democracy never come country lovingly oppressedout may th
making grand plans next week schoolwork spring break perfect world accomplish following
sort 
rm(corpus)
rm(corpus1)
rm(corpus2)
rm(corpus3)
rm(corpus4)
rm(corpus5)

Build N-gram

Extract the word and frequency of N-grams.

one<-function(x) NGramTokenizer(x,Weka_control(min=1,max=1))
two<-function(x) NGramTokenizer(x,Weka_control(min=2,max=2))
thr<-function(x) NGramTokenizer(x,Weka_control(min=3,max=3))

one_table<-TermDocumentMatrix(corpus6,control=list(tokenize=one))
two_table<-TermDocumentMatrix(corpus6,control=list(tokenize=two))
thr_table<-TermDocumentMatrix(corpus6,control=list(tokenize=thr))

one_corpus<-findFreqTerms(one_table,lowfreq=1000)
two_corpus<-findFreqTerms(two_table,lowfreq=80)
thr_corpus<-findFreqTerms(thr_table,lowfreq=10)

one_corpus_num<-rowSums(as.matrix(one_table[one_corpus,]))
one_corpus_table<-data.frame(Word=names(one_corpus_num),frequency=one_corpus_num)
one_corpus_sort<-one_corpus_table[order(-one_corpus_table$frequency),]
gt(head(one_corpus_sort))
Word frequency
just 12698
like 11136
will 10848
one 10586
can 9679
get 9286
 
two_corpus_num<-rowSums(as.matrix(two_table[two_corpus,]))
two_corpus_table<-data.frame(Word=names(two_corpus_num),frequency=two_corpus_num)
two_corpus_sort<-two_corpus_table[order(-two_corpus_table$frequency),]
gt(head(two_corpus_sort))
Word frequency
right now 1086
cant wait 861
dont know 859
last night 729
im going 649
feel like 557
 
thr_corpus_num<-rowSums(as.matrix(thr_table[thr_corpus,]))
thr_corpus_table<-data.frame(Word=names(thr_corpus_num),frequency=thr_corpus_num)
thr_corpus_sort<-thr_corpus_table[order(-thr_corpus_table$frequency),]
gt(head(thr_corpus_sort))
Word frequency
happy mothers day 169
cant wait see 168
let us know 122
im pretty sure 96
happy new year 85
new york city 61

Plot graph using ggplot

Using our N-grams; uni, bi & trigrams I plot the barplots for each classification.

one_g<-ggplot(one_corpus_sort[1:10,],aes(x=reorder(Word,-frequency),y=frequency,fill=frequency))
one_g<-one_g+geom_bar(stat="identity")
one_g<-one_g+labs(title="Unigrams",x="Words",y="Frequency")
one_g<-one_g+theme(axis.text.x=element_text(angle=90))
ggplotly(one_g)
two_g<-ggplot(two_corpus_sort[1:10,],aes(x=reorder(Word,-frequency),y=frequency,fill=frequency))
two_g<-two_g+geom_bar(stat="identity")
two_g<-two_g+labs(title="Bigrams",x="Words",y="Frequency")
two_g<-two_g+theme(axis.text.x=element_text(angle=90))
ggplotly(two_g)
thr_g<-ggplot(thr_corpus_sort[1:10,],aes(x=reorder(Word,-frequency),y=frequency,fill=frequency))
thr_g<-thr_g+geom_bar(stat="identity")
thr_g<-thr_g+labs(title="Trigrams",x="Words",y="Frequency")
thr_g<-thr_g+theme(axis.text.x=element_text(angle=90))
ggplotly(thr_g)