Introduction

The goal is to build a simple model for the relationship between words. This is the first step in building a text mining application able to predict the next word a user intends to type to aid in faster typing, similar to the feature currently embedded in most smartphones.

The data provided consists of 3 “corpora” of data, one collected from blog posts, one from news articles, and a third from twitter messages.

Exploratory Analysis

Loading the Data

The datasets were originally available in Finnish, German, Russian and English, but only the English datasets were used for this project.

Dataset: training data

I first load the required libraries,

library(NLP)
library(tm)
library(qdapRegex)
library(RWeka)
library(ggplot2)
library(grid)
library(gridExtra)

and then I read each corpus into R:

setwd("D:/Dean/R_courses/Capstone/final/en_US")
# Read the data
blogs <- readLines("en_US.blogs.txt")
news <- readLines("en_US.news.txt")
twitter <- readLines("en_US.twitter.txt")

A summary of the basic statistics of the files is reported below:

File Size (MB) N. of Lines N. of Words
en_US.blogs.txt 200 899,288 37,334,131
en_US.news.txt 196 1,010,242 34,372,530
en_US.twitter.txt 159 2,360,148 30,373,583

Sampling the data

Given the large size of the dataset, only a smaller sample of 30,000 lines will be analyzed for the exploratory analysis.

combinedText <- c(blogs, news, twitter)
set.seed(123)
sample.size<-30000
sample.blogs <- sample(blogs,sample.size)
sample.twitter <- sample(twitter,sample.size)
sample.news <- sample(news,sample.size)

# Clear the memory
rm(blogs, news, twitter, combinedText)

### Save sample
save(sample.blogs,sample.twitter,sample.news,file="samples.RData")

Cleaning the dataset

I will use the qdapRegex package to remove twitter retweets, @people, URLs, emoticons and hash tags.

# Remove twitter retweets 
sample.blogs <-  gsub("(RT|via)((?:\\b\\W*@\\w+)+)", "", sample.blogs)
sample.twitter <-  gsub("(RT|via)((?:\\b\\W*@\\w+)+)", "", sample.twitter)
sample.news <-  gsub("(RT|via)((?:\\b\\W*@\\w+)+)", "", sample.news)

# Remove @people
sample.blogs <-  gsub("@\\w+", "", sample.blogs)
sample.twitter <-  gsub("@\\w+", "", sample.twitter)
sample.news <-  gsub("@\\w+", "", sample.news)

# Remove http links
sample.blogs <-  gsub("http\\w+", "", sample.blogs)
sample.twitter <-  gsub("http\\w+", "", sample.twitter)
sample.news <-  gsub("http\\w+", "", sample.news)

# Remove emoticons (qdapRegex Package)
sample.blogs <- rm_emoticon(sample.blogs)
sample.twitter <-  rm_emoticon(sample.twitter)
sample.news <-  rm_emoticon(sample.news)

# Remove hash tags (qdapRegex Package)
sample.blogs <- rm_hash(sample.blogs)
sample.twitter <-  rm_hash(sample.twitter)
sample.news <-  rm_hash(sample.news)

# Remove urls (qdapRegex Package)
sample.blogs <- rm_url(sample.blogs)
sample.twitter <-  rm_url(sample.twitter)
sample.news <-  rm_url(sample.news)

I then employ the tm package for further cleaning:

# Convert samples into corpus format required by tm package
corpus.blogs <- Corpus(VectorSource(sample.blogs))
corpus.twitter <- Corpus(VectorSource(sample.twitter))
corpus.news <- Corpus(VectorSource(sample.news))

rm(sample.blogs,sample.twitter, sample.news)  # Clear memory

# Remove numbers
corpus.blogs <-tm_map(corpus.blogs, removeNumbers)
corpus.twitter <-tm_map(corpus.twitter, removeNumbers)
corpus.news <-tm_map(corpus.news, removeNumbers)

# Remove punctuation
corpus.blogs <-tm_map(corpus.blogs, removePunctuation)
corpus.twitter <-tm_map(corpus.twitter, removePunctuation)
corpus.news <-tm_map(corpus.news, removePunctuation)

# Remove unnecessary white space
corpus.blogs <-tm_map(corpus.blogs, stripWhitespace)
corpus.twitter <-tm_map(corpus.twitter, stripWhitespace)
corpus.news <-tm_map(corpus.news, stripWhitespace)

# Change to all lower-case
corpus.blogs <-tm_map(corpus.blogs, content_transformer(tolower))
corpus.twitter <-tm_map(corpus.twitter, content_transformer(tolower))
corpus.news <-tm_map(corpus.news, content_transformer(tolower))


#Remove swear/profanity words
if (!file.exists("badwords.txt")){
    download.file(url="https://raw.githubusercontent.com/shutterstock/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/en", destfile="badwords.txt")  
}
badWords <- readLines("badwords.txt")
corpus.blogs <-tm_map(corpus.blogs, removeWords, badWords)
corpus.twitter <-tm_map(corpus.twitter, removeWords, badWords)
corpus.news <-tm_map(corpus.news, removeWords, badWords)
rm(badwords)  # Clear memory

# Save the corpus
saveRDS(corpus.blogs, file = "corpusB.rds")
saveRDS(corpus.twitter, file = "corpusT.rds")
saveRDS(corpus.news, file = "corpusN.rds")

Tokenization and N-grams

Tokenization is the process of splitting a sentence into its tokens, where a token is a linguistic unit such as a word. Further analysis can then be performed using n-grams of word tokens, where an n-gram is a contiguous sequence of n words. More specifically, the NGramTokenizer function from the RWeka package will be used to build 1-gram (unigram), 2-gram (bigram) and 3-gram (trigram).

#  Load the corpus and convert them to dataframe.
corpus.blogs <- readRDS("corpusB.rds")
corpus.twitter <- readRDS("corpusT.rds")
corpus.news <- readRDS("corpusN.rds")

corpus.blogs <-data.frame(text=unlist(sapply(corpus.blogs,`[`, "content")), stringsAsFactors = FALSE)
corpus.twitter <-data.frame(text=unlist(sapply(corpus.twitter,`[`, "content")), stringsAsFactors = FALSE)
corpus.news <-data.frame(text=unlist(sapply(corpus.news,`[`, "content")), stringsAsFactors = FALSE)

## Function to derive n grams
ngramTokenizer <- function(theCorpus, ngramCount) {
    ngramFunction <- NGramTokenizer(theCorpus, 
                                    Weka_control(min = ngramCount, max = ngramCount, 
                                                 delimiters = " \\r\\n\\t.,;:\"()?!"))
    ngramFunction <- data.frame(table(ngramFunction))
    ngramFunction <- ngramFunction[order(ngramFunction$Freq, 
                                         decreasing = TRUE),]
    colnames(ngramFunction) <- c("String","Count")
    ngramFunction
}

# Generate unigrams and save
unigrams.blogs <- ngramTokenizer(corpus.blogs[1:nrow(corpus.blogs),], 1)
unigrams.twitter <- ngramTokenizer(corpus.twitter[1:nrow(corpus.twitter),], 1)
unigrams.news <- ngramTokenizer(corpus.news[1:nrow(corpus.news),], 1)
saveRDS(unigrams.blogs, file = "unigramsblogs.rds")
saveRDS(unigrams.twitter, file = "unigramstwitter.rds")
saveRDS(unigrams.news, file = "unigramsnews.rds")

# Generate bigrams and save
bigrams.blogs <- ngramTokenizer(corpus.blogs[1:nrow(corpus.blogs),], 2)
bigrams.twitter <- ngramTokenizer(corpus.twitter[1:nrow(corpus.twitter),], 2)
bigrams.news <- ngramTokenizer(corpus.news[1:nrow(corpus.news),], 2)
saveRDS(bigrams.blogs, file = "bigramsblogs.rds")
saveRDS(bigrams.twitter, file = "bigramstwitter.rds")
saveRDS(bigrams.news, file = "bigramsnews.rds")

# Generate trigrams and save
trigrams.blogs <- ngramTokenizer(corpus.blogs[1:nrow(corpus.blogs),], 3)
trigrams.twitter <- ngramTokenizer(corpus.twitter[1:nrow(corpus.twitter),], 3)
trigrams.news <- ngramTokenizer(corpus.news[1:nrow(corpus.news),], 3)
saveRDS(trigrams.blogs, file = "trigramsblogs.rds")
saveRDS(trigrams.twitter, file = "trigramstwitter.rds")
saveRDS(trigrams.news, file = "trigramsnews.rds")

Once the the n-grams are extracted, the top-30 n-gram phrases for blogs/Twitter/news are reported in the following plots:

plot_ngrams <- function(ngram_df, title) {
    ggplot(ngram_df[1:30,],
           aes(x = reorder(String, -Count), y = Count)) + 
        geom_bar(stat = "Identity", fill = "Sky Blue") +
        ggtitle(title) + xlab("Phrases") +
        theme(axis.text.x = element_text(angle = 45, hjust = 1))
}

# Plot the n-grams
p1=plot_ngrams(unigrams.blogs, "Top 30 unigrams - Blogs")
p2=plot_ngrams(bigrams.blogs, "Top 30 bigrams - Blogs")
p3=plot_ngrams(trigrams.blogs, "Top 30 trigrams - Blogs")
grid.arrange(p1, p2, p3, ncol=1)

p4=plot_ngrams(unigrams.twitter, "Top 30 unigrams - Twitter")
p5=plot_ngrams(bigrams.twitter, "Top 30 bigrams - Twitter")
p6=plot_ngrams(trigrams.twitter, "Top 30 trigrams - Twitter")
grid.arrange(p4, p5, p6, ncol=1)

p7=plot_ngrams(unigrams.news, "Top 30 unigrams - News")
p8=plot_ngrams(bigrams.news, "Top 30 bigrams - News")
p9=plot_ngrams(trigrams.news, "Top 30 trigrams - News")
grid.arrange(p7, p8, p9, ncol=1)

Conclusions and next steps

The initial results are promising and the following steps will be considered: