Exploratory Analysis

Before performing any exploratory analysis I did some cleaning of my data. This included creating sample sets of the blog, twitter, and news datasets that are a random 1% of the lines of data in those sets. I combined these data sets into a new sample dataset and created a corpus of this sample set along with clearing out punctuation, numbers, and white space.

For exploratory analysis, I utilized wordcloud and quanteda packages. I calculated single word frequencies, along with bigram and trigram frequencies. I also calculated word coverage and found the number of unique words needed for 50% coverage and 90% coverage. Additionally, I looked at these calculations using ggplot to create histograms of the bigram and trigram frequencies, highlighting the top 15 in both categories.

How I will use this data

I plan to utilize this data to create a next-word prediction application by creating a prediction algorithm and using this to create a Shiny application.

My coding so far

Below is the coding I have done thus far for this project:

# Week 1
# load data
blogsFileName <- "Coursera-SwiftKey/final/en_US/en_US.blogs.txt"
con <- file(blogsFileName, open = "r")
blogs <- readLines(con, encoding = "UTF-8", skipNul = TRUE)
close(con)
summary(blogs)
##    Length     Class      Mode 
##    899288 character character
newsFileName <- "Coursera-SwiftKey/final/en_US/en_US.news.txt"
con <- file(newsFileName, open = "r")
news <- readLines(con, encoding = "UTF-8", skipNul = TRUE)
close(con)
summary(news)
##    Length     Class      Mode 
##   1010206 character character
twitterFileName <- "Coursera-SwiftKey/final/en_US/en_US.twitter.txt"
con <- file(twitterFileName, open = "r")
twitter <- readLines(con, encoding = "UTF-8", skipNul = TRUE)
close(con)
summary(twitter)
##    Length     Class      Mode 
##   2360148 character character
rm(con)

# sampling, merge, and cleanup
library(tm)
## Warning: package 'tm' was built under R version 4.5.3
## Loading required package: NLP
## Warning: package 'NLP' was built under R version 4.5.3
library(slam)
## Warning: package 'slam' was built under R version 4.5.3
set.seed(123)
sampleBlogs <- sample(blogs, length(blogs) * 0.01, replace = FALSE)
sampleNews <- sample(news, length(news) * 0.01, replace = FALSE)
sampleTwitter <- sample(twitter, length(twitter) * 0.01, replace = FALSE)

sampleData <- c(sampleBlogs, sampleNews, sampleTwitter)
sampleDataFileName <- "Coursera-SwiftKey/final/en_US/en_US.sample.txt"
con <- file(sampleDataFileName, open = "w")
writeLines(sampleData, con)
close(con)

sampleData <- iconv(sampleData, "latin1", "ASCII", sub = "")
rm(blogs, news, twitter)

corpus <- VCorpus(VectorSource(sampleData))
corpus <- tm_map(corpus, content_transformer(tolower))
corpus <- tm_map(corpus, removePunctuation)
corpus <- tm_map(corpus, removeNumbers) 
corpus <- tm_map(corpus, stripWhitespace)

# Week 2
# exploratory analysis
library(wordcloud)
## Warning: package 'wordcloud' was built under R version 4.5.3
## Loading required package: RColorBrewer
library(quanteda)
## Warning: package 'quanteda' was built under R version 4.5.3
## Package version: 4.4
## Unicode version: 15.1
## ICU version: 74.1
## Parallel computing: 8 of 8 threads used.
## See https://quanteda.io for tutorials and examples.
## 
## Attaching package: 'quanteda'
## The following object is masked from 'package:tm':
## 
##     stopwords
## The following objects are masked from 'package:NLP':
## 
##     meta, meta<-
library(quanteda.textstats)
## Warning: package 'quanteda.textstats' was built under R version 4.5.3
tdm <- TermDocumentMatrix(corpus)
# word frequency
word_freq <- sort(row_sums(tdm), decreasing = TRUE)
head(word_freq)
##   the   and   for  that   you  with 
## 47728 24118 11131 10438  9285  7234
# two-word and three-word frequency
bigrams <- tokens(sampleData, remove_punct = TRUE, remove_numbers = TRUE) %>%
  tokens_tolower() %>%
  tokens_ngrams(n = 2) %>%
  dfm()

Freq2g <- head(textstat_frequency(bigrams))
top15_2gram <- Freq2g %>% head(15)

trigrams <- tokens(sampleData, remove_punct = TRUE, remove_numbers = TRUE) %>%
  tokens_tolower() %>%
  tokens_ngrams(n = 3) %>%
  dfm()

Freq3g <- head(textstat_frequency(trigrams))
top15_3gram <- Freq3g %>% head(15)

# word coverage
sorted_freq <- sort(row_sums(tdm), decreasing = TRUE)
cum_freq <- cumsum(sorted_freq) / sum(sorted_freq)
words_50 <- min(which(cum_freq >= 0.50))
words_90 <- min(which(cum_freq >= 0.90))
cat("Unique words for 50% coverage:", words_50, "\n")
## Unique words for 50% coverage: 316
cat("Unique words for 90% coverage:", words_90, "\n")
## Unique words for 90% coverage: 9896
# plots
library(ggplot2)
## 
## Attaching package: 'ggplot2'
## The following object is masked from 'package:NLP':
## 
##     annotate
plot_2gram <- ggplot(top15_2gram, aes(x = reorder(feature, frequency), y = frequency)) +
  geom_bar(stat = "identity", fill = "cyan") +
  coord_flip() +
  labs(
    title = "Top 15 Most Frequent Bigrams",
    subtitle = "From 1% Sampled Data (Blogs, News, Twitter)",
    x = "Bigram Phrase",
    y = "Frequency"
  ) +
  theme_minimal() +
  theme(axis.text.y = element_text(size = 11))
plot_2gram

plot_3gram <- ggplot(top15_3gram, aes(x = reorder(feature, frequency), y = frequency)) +
  geom_bar(stat = "identity", fill = "pink") + 
  coord_flip() +
  labs(
    title = "Top 15 Most Frequent Trigrams",
    subtitle = "From 1% Sampled Data (Blogs, News, Twitter)",
    x = "Trigram Phrase",
    y = "Frequency"
  ) +
  theme_minimal() +
  theme(axis.text.y = element_text(size = 11))
plot_3gram