Load the necessary libraries

library(rJava)
library(RWeka)
library(dplyr)
library(tm)
library(SnowballC)
library(ggplot2)
library(quanteda)

Read in data and take a random sample of each file

set.seed(123)
twitter <- readLines("en_US.twitter.txt")
n<- length(twitter)*0.01
indices <- sample(length(twitter), n)
twitter <- twitter[indices]

blog <- readLines("en_US.blogs.txt")
n <- length(blog)*0.01
indices <- sample(length(blog), n)
blog <- blog[indices]

news <- readLines("en_US.news.txt")
n <- length(news)*0.01
indices <- sample(length(news), n)
news <- news[indices]

Create a corpus and process the data (convert to lower case, remove punctuation and stop words, remove non-alphanumeric characters, perform word stemming).

corpus <- Corpus(VectorSource(c(twitter, news, blog)))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords, stopwords("english"))

remove_non_alphanumeric <- function(x) {
  x <- gsub("[^[:alnum:] ]", "", x)
  return(x)
}

corpus <- tm_map(corpus, content_transformer(remove_non_alphanumeric))

stemDocument <- function(x) {
  s <- wordStem(x)
  return(s)
}

corpus <- tm_map(corpus, content_transformer(stemDocument))

Create a document term matrix

dtm <- DocumentTermMatrix(corpus)
dtm_matrix <- as.matrix(dtm)

Determine the total number of unique words and the number of unique words covering 50% and 90$ of the cumulative frequency

word_counts <- colSums(dtm_matrix)
sorted_word_counts <- sort(word_counts, decreasing = T)
sorted_word_counts <- data.frame(words = names(sorted_word_counts), 
                                 count = sorted_word_counts)
nrow(sorted_word_counts)
## [1] 47485
sorted_word_counts$cumSum <- cumsum(sorted_word_counts$count)
which(sorted_word_counts$cumSum > 0.5 * nrow(sorted_word_counts))[1]
## [1] 15
which(sorted_word_counts$cumSum > 0.9 * nrow(sorted_word_counts))[1]
## [1] 36

47485 unique words, 15 words cover 50% of the cumulative frequency of all unique words, 36 words cover 90% of the cumulative frequency of all unique words. This shows us that the data is long tailed.

Next let’s plot the top 30 most common words

#single word counts
N <- 30
top_words <- sorted_word_counts[1:N, 1:2]

#graph single word counts
ggplot(top_words, aes(x = reorder(words, -count), y = count))+
  geom_bar(stat = "identity")+
  theme_classic()+
  labs(x = "Word", y = "Count")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

Next lets create a list of the most common bigrams

corpus_char <- sapply(corpus, as.character)
tokenized_corpus <- tokens(corpus_char)
bigrams <- tokens_ngrams(tokenized_corpus)
bigrams_list <- as.character(bigrams)
all_bigrams <- unlist(bigrams)
bigram_table <- table(all_bigrams)
bigrams_sorted <- sort(bigram_table, decreasing = T)
bigrams <- data.frame(Bigram = names(bigrams_sorted),
                      Count = as.numeric(bigrams_sorted),
                      stringsAsFactors = F)
bigrams <- bigrams %>%
  arrange(desc(Count))

and lets plot the most common bigrams

ggplot(bigrams[1:20,], aes(x = reorder(Bigram, -Count), y = Count))+
  geom_bar(stat = "identity")+
  theme_classic()+
  labs(x = "Counts", y = "Bigrams")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

lets have a look at trigrams

trigrams <- tokens_ngrams(tokenized_corpus, n = 3)
trigrams_list <- as.character(trigrams)
all_trigrams <- unlist(trigrams)
trigram_table <- table(all_trigrams)
trigrams_sorted <- sort(trigram_table, decreasing = T)
trigrams <- data.frame(Trigram = names(trigrams_sorted),
                      Count = as.numeric(trigrams_sorted),
                      stringsAsFactors = F)
trigrams <- trigrams %>%
  arrange(desc(Count))

and let’s plot the most common trigrams

ggplot(trigrams[1:20,], aes(x = reorder(Trigram, -Count), y = Count))+
  geom_bar(stat = "identity")+
  theme_classic()+
  labs(x = "Counts", y = "Bigrams")+
  theme(axis.text.x = element_text(angle = 45, hjust = 1))

In summary, although the corpus contains a large number of uique words, the data is long tailed and the vast majority of the cumulative frequency of unique words can be covered with a very small portion of words.There are several relatively common bigrams, but specific combinations of trigrams are less common.