Capstone Milestone 1

library(dplyr)

## Warning: package 'dplyr' was built under R version 4.5.2

## 
## Attaching package: 'dplyr'

## The following objects are masked from 'package:stats':
## 
##     filter, lag

## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union

library(tidytext)
library(ggplot2)

## Warning: package 'ggplot2' was built under R version 4.5.2

library(ggridges)
library(stringr)
library(tidyr)

#Loading Files

# Path to your top-level folder
root <- "~/Desktop/Coursera/Data_sci_capstone/final"

# Find all .txt files in all subfolders
files <- list.files(
  path = root,
  #using regex to find .txt files start with en_US,  '\\.' look for a literal period and 'txt$' looks fot txt at the end of the string
  pattern = "^en_US.*\\.txt$$",
  #recursive = TRUE is for looking in the subfolders of the root directory
  recursive = TRUE,
  #the final names of the files will be the full paths to them for readLines() later
  full.names = TRUE,
  #ignore the case of the naming
  ignore.case = TRUE
)




#lapply() to loop over every element in the list
#function(f) takes each file as an argument
#run through readLines to get a character vector for each in the file
texts <- lapply(files, function(f) readLines(f, warn = FALSE))

#label each text with its file path
names(texts) <- files

#Exploratory Data Analysis

The goal of exploratory analysis will be to identify and visualize which individual words, pairs, and triplets (or ngrams) are most common in the data from each individual source. The most common ngrams’ positions within sentences will also be visualized. The same process will be followed with punctiuation: identify the most common ones and where they occur in sentences.

#sampling just 5% of each document so it runs faster

set.seed(123)

texts <- lapply(texts, function(x) {
  n_keep <- ceiling(length(x) * 0.05)
  x[sample(length(x), n_keep)]
})


sapply(texts, length)

##   /Users/austenjack/Desktop/Coursera/Data_sci_capstone/final/en_US/en_US.blogs.txt 
##                                                                              44965 
##    /Users/austenjack/Desktop/Coursera/Data_sci_capstone/final/en_US/en_US.news.txt 
##                                                                              50513 
## /Users/austenjack/Desktop/Coursera/Data_sci_capstone/final/en_US/en_US.twitter.txt 
##                                                                             118008

This is an initialization of the data for exploratory analysis. For any given ngram, gather total frequency and sentence positions.

#making a table of all lines in all texts, adding a column that notes which source a line came from
lines_tbl <- bind_rows(lapply(texts, \(x) tibble(text = x)), .id = "source") %>%
  mutate(source = basename(source))


#tokenizing the sentences in the lines_tbl table
sentences <- lines_tbl %>%
  unnest_tokens(sentence, text, token = "sentences", to_lower = FALSE) %>%
  mutate(sentence_id = row_number())


#making a function that will tokenize the words of sentences at different n-grams (single word, paris, triplets, etx)
ngram_tbl <- function(sentences, k, ngram_name) {
  #take the sentences tokens df
  sentences %>%
    #tokenize ngrams based on argument k
    unnest_tokens(ngram, sentence, token = "ngrams", n = k) %>%
    #remove na's
    filter(!is.na(ngram)) %>%                  
    #group tokenized ngrams by which sentence they came from
    group_by(sentence_id) %>%
    #for each token add their position within their sentence
    mutate(position = row_number(),
           #also add the relative position within their sentence (.25, .5, etc)
           rel_position = position / n()) %>%
    #ungroup so the df is back to a normal df
    ungroup()
}


#get dfs with individual words, pairs, and triplets including which source they came from and their positions within a sentence
words <- ngram_tbl(sentences, 1)
pairs <- ngram_tbl(sentences, 2)
trips <- ngram_tbl(sentences, 3)

#get frequencies come from the same table, add raw counts and counts per 100 ngrams
words_freq <- count(words, ngram, sort = TRUE) %>% mutate(per_100 = 100 * n / sum(n))
pairs_freq <- count(pairs, ngram, sort = TRUE) %>% mutate(per_100 = 100 * n / sum(n))
trips_freq <- count(trips, ngram, sort = TRUE) %>% mutate(per_100 = 100 * n / sum(n))

With the initialized data create summary stats of words for each source text. Total sentence count, word count, character count, average sentence length, total number of unique words, words per sentence, characters per word, etc.

#corpus summary table

#function for making a summary table for the text documents
corpus_summary <- function(lines_tbl, sentences, words) {
  #with the table of lines (from before) make stats for each source
  line_stats <- lines_tbl %>%
    
    #group the lines table y source
    group_by(source) %>%
    
    #summarise total number of lines, total number of characters for each source
    summarise(lines = n(),
              characters = sum(str_length(text)),
              .groups = "drop")

  #get total number of sentences for each source
  sentence_stats <- sentences %>%
    count(source, name = "sentences")

  #summarise word stats for each source
  word_stats <- words %>%
    group_by(source) %>%
    #get total number words, total unique words
    summarise(words = n(),
              unique_words = n_distinct(ngram),
              .groups = "drop")

  #put all the stats together to make a summary table
  line_stats %>%
    left_join(sentence_stats, by = "source") %>%
    left_join(word_stats, by = "source") %>%
    #add other stats calculated by the ones collected: words_per_line, words_per_sentence, chars_per_word
    mutate(words_per_line = round(words / lines, 1),
           words_per_sentence = round(words / sentences, 1),
           chars_per_word = round(characters / words, 1))
}

#get initial summary table using the function
summary_tbl <- corpus_summary(lines_tbl, sentences, words)


#want to add on a final row that has stats for all texts combined

#get total words in all texts
n_words  <- nrow(words)
#get total unique words in all texts
n_unique <- n_distinct(words$ngram)

#manually make the final row of summary stats with the same column names as the full summary table
total_row <- tibble(
  source       = "All sources",
  lines        = nrow(lines_tbl),
  characters   = sum(str_length(lines_tbl$text)),
  sentences    = nrow(sentences),
  words        = n_words,
  unique_words = n_unique
) %>%
  mutate(words_per_line     = round(words / lines, 1),
         words_per_sentence = round(words / sentences, 1),
         chars_per_word     = round(characters / words, 1))

#bind the final row to the table
summary_tbl <- bind_rows(summary_tbl, total_row)

summary_tbl

Things to note: -twitter has fewer words per line and sentence than blog posts or news articles -all sources have extremely similar unique word count and characters per word

We have the summary statistics for each source, now will zoom in to get the frequencies of the 3 main ngrams being analyzed: individual words, pairs, and triplets.

#top_positions() will make a new table with only the most frequent words and their positions within sentences

top_positions <- function(pos, freq, n=10){
  #use frequency table to get most frequent ngrams
  
  
  n_top <- n
  
  
  top_ngrams <- freq %>%
    slice_max(n, n = n_top, with_ties = FALSE) %>%
    #pull(ngram) returns the column as a vector, so after slicing it will pull the 'word' column with the top 10 words
    pull(ngram)
  
  
  #table with position values from before
  pos %>%
  #filter to only have the most frequent ngram
  filter(ngram %in% top_ngrams) %>%
  #group by ngrams to get stats on individual ngrams
  group_by(ngram) %>%
  #want total counts of ngram, average sentence position, median sentence position, average relative position
  summarise(occurrences = n(),
            mean_pos = mean(position),
            median_pos = median(position),
            mean_rel_pos = mean(rel_position)) %>%
  #join the per_100 value calculated before
  left_join(select(freq, ngram, per_100), by = "ngram") %>%
  #arrange so most frequent words are first
  arrange(desc(occurrences))
  
}

#get the top 10 ngrams of individual words, pairs, and triplets with their sentence positions
top_words <- top_positions(words, words_freq, 10)
top_pairs <- top_positions(pairs, pairs_freq, 10)
top_trips <- top_positions(trips, trips_freq, 10)

#plot each most common ngram's frequency using their counts per 100 ngrams

top_per_100_plot <- function(top_ngram, type){
  ggplot(top_ngram, aes(reorder(ngram, -per_100), per_100)) +
  geom_col() +
  labs(x = type, y = paste("occurrences per 100 ", type, "s"))
}

Bar plots of each ngram dataset most common ngram frequency per 100 ngrams

top_words_plot <- top_per_100_plot(top_words, 'word')
top_words_plot

top_pairs_plot <- top_per_100_plot(top_pairs, 'pair')
top_pairs_plot

top_trips_plot <- top_per_100_plot(top_trips, 'triplet')
top_trips_plot

Notes: -“the” is the most common word used by far -“the” can also be found in many of the most common pairs and triplets

#make similar plot but faceted by source

top_freq_by_source_plot <- function(tbl, type, n_top = 10) {

  tbl %>%
    count(source, ngram) %>%
    group_by(source) %>%
    mutate(per_100 = 100 * n / sum(n)) %>%
    slice_max(n, n = n_top, with_ties = FALSE) %>%
    ungroup() %>%
    ggplot(aes(reorder_within(ngram, per_100, source), per_100)) +
    geom_col() +
    scale_x_reordered() +
    facet_wrap(~ source, scales = "free_y") +
    coord_flip() +
    labs(x = type, y = paste0("occurrences per 100 ", type, "s"))
}

Bar plots of the most common ngrams across the three different sources

top_freq_by_source_plot(words, "word", 10)

top_freq_by_source_plot(pairs, "pair", 10)

top_freq_by_source_plot(trips, "triplet", 10)

Notes: -“the” is most common in all three sources -the sources generally have the same top 10 most common words

Now visualizing distributions of the most common ngrams in sentences where they appear.

#function for making plots that show disributions of word locations in sentences
#arguments are title for the plot, the df with position data, df with frequency/counts data, and how many top ngrams to plot
top_sentence_dist_plot <- function(title, pos, freq, n = 10) {
  #get top ngrams as have done before
  n_top <- n
  top_ngrams <- freq %>%
    slice_max(.data$n, n = n_top, with_ties = FALSE) %>%
    pull(ngram)

  
  #with the position df to plot relative position of top ngrams in density plots
  pos %>%
    filter(ngram %in% top_ngrams) %>%
    #rev() makes the most frequent ngram appears at the top of the plot
    mutate(ngram = factor(ngram, levels = rev(top_ngrams))) %>%
    ggplot(aes(x = rel_position, y = ngram, fill = ngram)) +
    geom_density_ridges(
      alpha = 0.8,
      scale = 1.5,           
      rel_min_height = 0.01
    ) +
    scale_fill_viridis_d() +
    scale_x_continuous(limits = c(0, 1), expand = c(0, 0)) +
    labs(title = title, x = "Relative position", y = NULL) +
    theme_ridges() +
    theme(legend.position = "none")
}

top_sentence_dist_plot("Sentence Distribution of Most Common Words",words, words_freq, 10)

## Picking joint bandwidth of 0.024

top_sentence_dist_plot("Sentence Distribution of Most Common Pairs", pairs, pairs_freq, 10)

## Picking joint bandwidth of 0.0386

top_sentence_dist_plot("Sentence Distribution of Most Common Triplets",trips, trips_freq, 10)

## Picking joint bandwidth of 0.0586

Notes: -individual words generally have uniform use throughout sentences -the word “I” is more frequent at the beginning of a sentence -the most common pairs have higher frequencies toward the end of sentences -most common triplets have varying distributions

Doing the same thing but for each source.

#make same plots as before but facet by source
top_sentence_dist_plot_source <- function(pos, freq, n = 10) {
  n_top <- n
  top_ngrams <- freq %>%
    slice_max(.data$n, n = n_top, with_ties = FALSE) %>%
    pull(ngram)

  pos %>%
    filter(ngram %in% top_ngrams) %>%
    mutate(ngram = factor(ngram, levels = rev(top_ngrams))) %>%
    ggplot(aes(x = rel_position, y = ngram, fill = ngram)) +
    geom_density_ridges(alpha = 0.8, scale = 1.5, rel_min_height = 0.01) +
    scale_fill_viridis_d() +
    scale_x_continuous(limits = c(0, 1), expand = c(0, 0)) +
    facet_wrap(~ source) +
    labs(x = "Relative position", y = NULL) +
    theme_ridges() +
    theme(legend.position = "none")
}

top_sentence_dist_plot_source(pos = words, freq = words_freq, 10)

## Picking joint bandwidth of 0.0295

## Picking joint bandwidth of 0.0309

## Picking joint bandwidth of 0.0295

top_sentence_dist_plot_source(pos = pairs, freq = pairs_freq, 10)

## Picking joint bandwidth of 0.048

## Picking joint bandwidth of 0.0477

## Picking joint bandwidth of 0.0476

top_sentence_dist_plot_source(pos = trips, freq = trips_freq, 10)

## Picking joint bandwidth of 0.0828

## Picking joint bandwidth of 0.0797

## Picking joint bandwidth of 0.0715

Notes: -individual ngrams -the distributions of single word ngrams in sentences are consistent across sources, but twitter has slightly weird distributions likely due to short sentences -pairs -most common pairs also have similar distributions -triplets -most common triplets seem to have slighly different distributions across the different sources

Getting the stats for punctuation as with single word ngrams

#punctuation

punct_tbl <- lines_tbl %>%
  mutate(punct = str_extract_all(text, "[\\p{P}--['’]]")) %>%
  select(source, punct) %>%
  unnest(punct)

punct_freq <- count(punct_tbl, punct, sort = TRUE) %>%
  mutate(per_100 = 100 * n / sum(n))

punct_freq

Punctuation frequency across different sources

ggplot(punct_freq, aes(reorder(punct, -n), n)) +
  geom_col() +
  labs(x = "punctuation mark", y = "count")

# by source, as a share of each source's punctuation
punct_tbl %>%
  count(source, punct) %>%
  group_by(source) %>%
  mutate(per_100 = 100 * n / sum(n)) %>%
  slice_max(n, n = 10, with_ties = FALSE) %>%
  ungroup() %>%
  ggplot(aes(reorder_within(punct, per_100, source), per_100)) +
  geom_col() +
  scale_x_reordered() +
  facet_wrap(~ source, scales = "free_y") +
  coord_flip() +
  labs(x = "punctuation mark", y = "per 100 punctuation marks")

Notes: -Twitter generally has higher rates of punctuation use thatn blogs and news articles -the overall distributions of the frequencies follow a similar shape for all three sources

punct_pattern <- "[\\p{P}--['’]]"

punct_pos <- sentences %>%
  mutate(len = str_length(sentence),
         loc = lapply(str_locate_all(sentence, punct_pattern), as.data.frame)) %>%
  select(source, sentence_id, sentence, len, loc) %>%
  unnest(loc) %>%
  mutate(ngram = str_sub(sentence, start, end),  
         position = start,
         rel_position = start / len) %>%
  select(source, sentence_id, ngram, position, rel_position)

punct_freq <- count(punct_pos, ngram, sort = TRUE) %>%
  mutate(per_100 = 100 * n / sum(n))

Sentence distribution plots of most common punctuation and across different sources

#sentence distributions of punctuation
top_sentence_dist_plot("Sentence Distribution of Most Common Punctuation", punct_pos, punct_freq, 10)

## Picking joint bandwidth of 0.0248

top_sentence_dist_plot_source(pos = punct_pos, freq = punct_freq, 10)

## Picking joint bandwidth of 0.0375

## Picking joint bandwidth of 0.0379

## Picking joint bandwidth of 0.0291

Notes: -punctutaion marks show exaclty where we’d expect them to in sentences -distributions in the three sources follow each other closely