Load and Clean Data

The data consists in a set of texts from twitter, blogs and news. For the reading of the data I used the data on the zip file downloaded from [link] (https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip). This file contains the three files mentioned above. For this report I am using data saved from the cache in order to accelerate the process. Then the data was turned into dataframes to work with dplyr and tidyr functions. All numbers and punctuation marks were removed. Then the data was tokenized to handle the data for the analysis of each word and all data was changed to lowercase. In order to eliminate unwanted words, a data frame from the tidytext library called “stop_words” was used and addtional words were added considering some bad words seen on the exploratory analysis and fractions of contracted words.

library(doParallel)
cl <- makePSOCKcluster(30)
registerDoParallel(cl)
#Load Libraries needed
library(dplyr)
library(tidyr)
library(tidytext)
library(ggplot2)
library(stringr)
library(scales)
library(tm)
library(gridExtra)
library(wordcloud)
library(DT)

# #----------------------------------------
# #Load Data
# #----------------------------------------
# url_data<-"https://d396qusza40orc.cloudfront.net/dsscapstone/dataset/Coursera-SwiftKey.zip"
# 
# # create a temporary directory
# td = tempdir()
# # create the placeholder file
# tf = tempfile(tmpdir=td, fileext=".zip")
# # download into the placeholder file
# download.file(url_data, tf)
# #close(tf)
# 
# twitter<- readLines(unz(tf, "final/en_US/en_US.twitter.txt"),encoding = "UTF-8", skipNul = TRUE)
# blogs  <- readLines(unz(tf, "final/en_US/en_US.blogs.txt"),encoding = "UTF-8", skipNul = TRUE)
# news   <- readLines(unz(tf, "final/en_US/en_US.news.txt"),encoding = "UTF-8", skipNul = TRUE)
# 
# save(twitter, file="./cache/twitter.RData")
# save( blogs , file="./cache/blogs.RData")
# save( news  , file="./cache/news.RData")


#----------------------------------------
load("./cache/twitter.RData")
load("./cache/blogs.RData")
load("./cache/news.RData")
#----------------------------------------
#----------------------------------------
#Sample data
#----------------------------------------
# set.seed(140517)
# twitter<- sample(twitter,round(length(twitter)/3),replace=FALSE)
# blogs <- sample(blogs,round(length(blogs)/3),replace=FALSE)
# news <- sample(news,round(length(news)/3),replace=FALSE)

#----------------------------------------
#Replace punctuation marks with space
#----------------------------------------
twitter <- gsub("[[:punct:]]"," ",twitter)
blogs <- gsub("[[:punct:]]"," ",blogs)
news <- gsub("[[:punct:]]"," ",news)

twitter <- gsub("[0-9]","",twitter)
blogs <- gsub("[0-9]","",blogs)
news <- gsub("[0-9]","",news)

#----------------------------------------
#Turn into dataframe and Tokenize
#----------------------------------------
twt <- data_frame(line=1:length( twitter ),text= twitter )
twt1<- twt %>% unnest_tokens(word, text)

blg <- data_frame(line=1:length( blogs ),text= blogs )
blg1<- blg %>% unnest_tokens(word, text)

new <- data_frame(line=1:length( news ),text= news )
new1<- new %>% unnest_tokens(word, text)

rm(twitter,blogs,news)
#----------------------------------------
#Remove unwanted words
#----------------------------------------
data(stop_words);words<-c("rt","ll","ve","im","don","doesn","didn","haha","shit","ass","fuck","st")
stop_words<-stop_words%>%bind_rows(data.frame(word=words,lexicon="Extra"))#%>%filter(!word=="i")

twt1<- twt1 %>%  anti_join(stop_words)
## Joining, by = "word"
blg1<- blg1 %>%  anti_join(stop_words)
## Joining, by = "word"
new1<- new1 %>%  anti_join(stop_words)
## Joining, by = "word"

Exploratory analysis

Data description

The number of lines of each data set is on the next table.

dat<-c("Twitter","Blogs","News")
data.frame(Data=dat,NumLines=c(nrow(twt),nrow(blg),nrow(new)))
##      Data NumLines
## 1 Twitter  2360148
## 2   Blogs   899288
## 3    News  1010242

The number of words of each data set is on the next table.

dat<-c("Twitter","Blogs","News")
data.frame(Data=dat,NumWords=c(length(unique(twt1$word)),length(unique(blg1$word)),length(unique(new1$word))))
##      Data NumWords
## 1 Twitter   305506
## 2   Blogs   258024
## 3    News   214358

Summary

Most frequent words are shown in the next table, for twitter, blogs and news respectively. Additionaly wordclouds were graphed for each dataset.

#-----------------------------------
#Exploratory analysis
#-----------------------------------
datatable(head(twt1 %>%  count(word, sort = TRUE) ,14))
datatable(head(blg1 %>%  count(word, sort = TRUE) ,14))
datatable(head(new1 %>%  count(word, sort = TRUE) ,14))
twt1 %>%
  count(word) %>%
  with(wordcloud(word, n, max.words = 100))

blg1 %>%
  count(word) %>%
  with(wordcloud(word, n, max.words = 100))

new1 %>%
  count(word) %>%
  with(wordcloud(word, n, max.words = 100))

n-Grams

Two-grams and tri-grams were considered to plot the most frequent ones.

Twitter

twt_bigrams <- twt %>%
  unnest_tokens(bigram, text, token = "ngrams", n = 2)
twt_bigrams %>%
  count(bigram, sort = TRUE) %>%
  filter(n > 30000) %>%
  mutate(bigram = reorder(bigram, n)) %>%
  ggplot(aes(bigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

twt_trigrams <- twt %>%
  unnest_tokens(trigram, text, token = "ngrams", n = 3)
twt_trigrams %>%
  count(trigram, sort = TRUE) %>%
  filter(n > 5000) %>%
  mutate(trigram = reorder(trigram, n)) %>%
  ggplot(aes(trigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

Blogs

blg_bigrams <- blg %>%
  unnest_tokens(bigram, text, token = "ngrams", n = 2)
blg_bigrams %>%
  count(bigram, sort = TRUE) %>%
  filter(n > 30000) %>%
  mutate(bigram = reorder(bigram, n)) %>%
  ggplot(aes(bigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

blg_trigrams <- blg %>%
  unnest_tokens(trigram, text, token = "ngrams", n = 3)
blg_trigrams %>%
  count(trigram, sort = TRUE) %>%
  filter(n > 5000) %>%
  mutate(trigram = reorder(trigram, n)) %>%
  ggplot(aes(trigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

News

new_bigrams <- new %>%
  unnest_tokens(bigram, text, token = "ngrams", n = 2)
new_bigrams %>%
  count(bigram, sort = TRUE) %>%
  filter(n > 30000) %>%
  mutate(bigram = reorder(bigram, n)) %>%
  ggplot(aes(bigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

new_trigrams <- new %>%
  unnest_tokens(trigram, text, token = "ngrams", n = 3)
new_trigrams %>%
  count(trigram, sort = TRUE) %>%
  filter(n > 5000) %>%
  mutate(trigram = reorder(trigram, n)) %>%
  ggplot(aes(trigram, n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip()

This information gives an idea of how to work with the data and the use of the n-grams to do the prediction of the next word that will be typed next. In order to detect other relevant n-grams, for the final modelling the dataframe for “stop_words” will be used on the n-grams because as can be seen from the graphs a lot of “in the”, “to the”, “on the” are shown so maybe another interesting combinations show up.