Load the necessary libraries
library(rJava)
library(RWeka)
library(dplyr)
library(tm)
library(SnowballC)
library(ggplot2)
library(quanteda)
Read in data and take a random sample of each file
set.seed(123)
twitter <- readLines("en_US.twitter.txt")
n<- length(twitter)*0.01
indices <- sample(length(twitter), n)
twitter <- twitter[indices]
blog <- readLines("en_US.blogs.txt")
n <- length(blog)*0.01
indices <- sample(length(blog), n)
blog <- blog[indices]
news <- readLines("en_US.news.txt")
n <- length(news)*0.01
indices <- sample(length(news), n)
news <- news[indices]
Create a corpus and process the data (convert to lower case, remove punctuation and stop words, remove non-alphanumeric characters, perform word stemming).
corpus <- Corpus(VectorSource(c(twitter, news, blog)))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeWords, stopwords("english"))
remove_non_alphanumeric <- function(x) {
x <- gsub("[^[:alnum:] ]", "", x)
return(x)
}
corpus <- tm_map(corpus, content_transformer(remove_non_alphanumeric))
stemDocument <- function(x) {
s <- wordStem(x)
return(s)
}
corpus <- tm_map(corpus, content_transformer(stemDocument))
Create a document term matrix
dtm <- DocumentTermMatrix(corpus)
dtm_matrix <- as.matrix(dtm)
Determine the total number of unique words and the number of unique words covering 50% and 90$ of the cumulative frequency
word_counts <- colSums(dtm_matrix)
sorted_word_counts <- sort(word_counts, decreasing = T)
sorted_word_counts <- data.frame(words = names(sorted_word_counts),
count = sorted_word_counts)
nrow(sorted_word_counts)
## [1] 47485
sorted_word_counts$cumSum <- cumsum(sorted_word_counts$count)
which(sorted_word_counts$cumSum > 0.5 * nrow(sorted_word_counts))[1]
## [1] 15
which(sorted_word_counts$cumSum > 0.9 * nrow(sorted_word_counts))[1]
## [1] 36
47485 unique words, 15 words cover 50% of the cumulative frequency of all unique words, 36 words cover 90% of the cumulative frequency of all unique words. This shows us that the data is long tailed.
Next let’s plot the top 30 most common words
#single word counts
N <- 30
top_words <- sorted_word_counts[1:N, 1:2]
#graph single word counts
ggplot(top_words, aes(x = reorder(words, -count), y = count))+
geom_bar(stat = "identity")+
theme_classic()+
labs(x = "Word", y = "Count")+
theme(axis.text.x = element_text(angle = 45, hjust = 1))
Next lets create a list of the most common bigrams
corpus_char <- sapply(corpus, as.character)
tokenized_corpus <- tokens(corpus_char)
bigrams <- tokens_ngrams(tokenized_corpus)
bigrams_list <- as.character(bigrams)
all_bigrams <- unlist(bigrams)
bigram_table <- table(all_bigrams)
bigrams_sorted <- sort(bigram_table, decreasing = T)
bigrams <- data.frame(Bigram = names(bigrams_sorted),
Count = as.numeric(bigrams_sorted),
stringsAsFactors = F)
bigrams <- bigrams %>%
arrange(desc(Count))
and lets plot the most common bigrams
ggplot(bigrams[1:20,], aes(x = reorder(Bigram, -Count), y = Count))+
geom_bar(stat = "identity")+
theme_classic()+
labs(x = "Counts", y = "Bigrams")+
theme(axis.text.x = element_text(angle = 45, hjust = 1))
lets have a look at trigrams
trigrams <- tokens_ngrams(tokenized_corpus, n = 3)
trigrams_list <- as.character(trigrams)
all_trigrams <- unlist(trigrams)
trigram_table <- table(all_trigrams)
trigrams_sorted <- sort(trigram_table, decreasing = T)
trigrams <- data.frame(Trigram = names(trigrams_sorted),
Count = as.numeric(trigrams_sorted),
stringsAsFactors = F)
trigrams <- trigrams %>%
arrange(desc(Count))
and let’s plot the most common trigrams
ggplot(trigrams[1:20,], aes(x = reorder(Trigram, -Count), y = Count))+
geom_bar(stat = "identity")+
theme_classic()+
labs(x = "Counts", y = "Bigrams")+
theme(axis.text.x = element_text(angle = 45, hjust = 1))
In summary, although the corpus contains a large number of uique words, the data is long tailed and the vast majority of the cumulative frequency of unique words can be covered with a very small portion of words.There are several relatively common bigrams, but specific combinations of trigrams are less common.