Summary This brief overview examines the three en_US data sets, including data collected from blogs, news websites, and twitter. These data sets vary in both size and structure, with Twitter containing shorter text entries compared to news websites and blogs. Summary statistics are included in Table 1, including the total lines, total words, total characters, and mean words per line in each dataset. Basic plots are also presented for these data.

I have also included a preliminary analysis of word count frequencies across the three data sets. The top 20 words in each data set, as well as their frequency, are included below. These preliminary findings suggest that care must be taken to account for boht word frequency and context when designing the algorithm.

Goals for Prediction Algorithm These preliminary results will be used to guide the development of an app and algorithm for word prediction. In the app, a user will enter a sequence of words, and an n-gram model will be used to predict the next word. This model will be trained on data from US news, blogs, and twitter data, and will need the flexibility to handle user inputs that are not observed in the training datasets. To test the accuracy of the model, the en_US data will be subset into training (80%) and test (20%) datasets. The final model will balance accuracy and computational speed in the Shiny environment.

Load data

# Load files
twitter.path <- "C:/Users/eamon/Google Drive/Postdoc/Coursera/DataScienceSpecialization/Capstone/en_US.twitter.txt"
us.twitter1 <- readLines(twitter.path, n = -1, skipNul = TRUE)


blogs.path <- "C:/Users/eamon/Google Drive/Postdoc/Coursera/DataScienceSpecialization/Capstone/en_US.blogs.txt"
us.blogs1 <- readLines(blogs.path, n= -1, skipNul = TRUE)

news.path <- "C:/Users/eamon/Google Drive/Postdoc/Coursera/DataScienceSpecialization/Capstone/en_US.news.txt"
us.news1 <- readLines(news.path, n= -1, skipNul = TRUE)

# Remove unnecessary objects
rm(blogs.path, twitter.path, news.path)

Summary Measures: Line, word, and character counts

#########################   Calculate Summary Measures   ##################
# Custom function to count lines, words, and characters within a text file
text.fun <- function(x) {
  word.str <- strsplit(x, "\\s+") # create string of words split by white spaces
  line.length <- sapply(word.str, length)
  
  data.frame(Lines = length(x),
             Words = sum(sapply(word.str, length)),
             Characters = sum(nchar(x)),
             MeanWordsPerLine = round(mean(line.length)))
}



# Make summary data frame
summary.table <- rbind(Blogs = text.fun(us.blogs1),
                       News = text.fun(us.news1),
                       Twitter = text.fun(us.twitter1))

summary.table2 <- tibble::rownames_to_column(summary.table, var = "Dataset")



###########################   Table 1: Summary Stats   #################3
summary.table3 <- summary.table2 %>%
  kable(align="c",
  caption = "Table 1: Summary statistics for Blog, News, and Twitter data") %>%
kable_styling(bootstrap_options = c("striped","hover"), position = "center")

# remove unnecessary objects
rm(summary.table)

summary.table3
Table 1: Summary statistics for Blog, News, and Twitter data
Dataset Lines Words Characters MeanWordsPerLine
Blogs 899288 37334131 206824509 42
News 1010206 34371031 203214543 34
Twitter 2360148 30373583 162122861 13 
##############################   Summary Figures   #######################

# Lines per dataset
line.fig <- ggplot(summary.table2, aes(x = Dataset, y = Lines, fill = Dataset)) +
  geom_col() +
  labs(title = "Number of Lines per Dataset",
       x = "Dataset",
       y = "Number of Lines") +
  theme_bw() +
  theme(legend.position = "none")

line.fig

# Words per dataset
word.fig <- ggplot(summary.table2, aes(x = Dataset, y = Words, fill = Dataset)) +
  geom_col() +
  labs(title = "Number of Words per Dataset",
       x = "Dataset",
       y = "Number of Words") +
  theme_bw() +
  theme(legend.position = "none")

word.fig

# Characters per dataset
char.fig <- ggplot(summary.table2, aes(x = Dataset, y = Characters, fill = Dataset)) +
  geom_col() +
  labs(title = "Number of Characters per Dataset",
       x = "Dataset",
       y = "Number of Characters") +
  theme_bw() +
  theme(legend.position = "none")

char.fig

Word Frequency

# Extract words from each dataset
blogs.words <- unlist(strsplit(us.blogs1, "\\s+"))
news.words <- unlist(strsplit(us.news1, "\\s+"))
twitter.words <- unlist(strsplit(us.twitter1, "\\s+"))

# Count words
blogs.freq <- head(sort(table(blogs.words), decreasing = TRUE), 20)
news.freq <- head(sort(table(news.words), decreasing = TRUE), 20)
twitter.freq <- head(sort(table(twitter.words), decreasing = TRUE), 20)

# Print the frequency of the top 20 words in each data set
blogs.freq
## blogs.words
##     the      to     and      of       a       I      in    that      is     for 
## 1659151 1043878 1015714  862906  857102  738534  540436  421628  412438  337156 
##     was    with      it      on      my     you    have      be      as    this 
##  271439  271302  270280  252275  239952  238652  210982  198728  196879  188536
news.freq
## news.words
##     the      to     and       a      of      in     for    that      is      on 
## 1712353  889203  845199  826700  766123  621911  332309  314011  275113  249810 
##    with     The     was      at      as      he      be     his    from      it 
##  242654  225941  225554  198235  170983  166954  148131  147552  147344  143295
twitter.freq
## twitter.words
##    the     to      I      a    you    and    for     of     in     is     on 
## 837023 761902 604531 572691 416377 397642 368422 349367 348815 329396 253558 
##     my     it   that     be     at   with   your   have     me 
## 248739 192437 190845 172887 171759 163808 157114 149376 143522