CH3 Analyzing word and document frequency: tf-idf

library(data.table)
library(data.table)
library(dplyr) #data manipulation
library(ggplot2) #visualizations
library(gridExtra) #viewing multiple plots together
library(tidytext) #text mining
#library(wordcloud2) #creative visualizations

load data

lyric_data=fread('../data/prince_raw_data.csv')

3.1 Term frequency

library(dplyr)
library(janeaustenr)
library(tidytext)
lyric_words <- lyric_data[,1:6] %>%
  unnest_tokens(word, text) %>%
  count(album, word, sort = TRUE) %>%
  ungroup()
total_words <- lyric_words %>% 
  group_by(album) %>% 
  summarize(total = sum(n))
lyric_words <- left_join(lyric_words, total_words)

Joining, by = "album"

lyric_words

library(ggplot2)
lyric_words_4albums<-lyric_words %>%
  filter(.,album %in% 
           c('Other Songs',
             'Emancipation',
             'Crystal Ball',
             'Crystal Ball'))
ggplot(lyric_words_4albums, aes(n/total, fill = album)) +
  geom_histogram(show.legend = FALSE) +
  xlim(NA, 0.0009) +
  facet_wrap(~album, ncol = 2, scales = "free_y")

3.2 Zipf’s law

freq_by_rank <- lyric_words_4albums %>% 
  group_by(album) %>% 
  mutate(rank = row_number(), 
         `term frequency` = n/total)
freq_by_rank

freq_by_rank %>% 
  ggplot(aes(rank, `term frequency`, color = album)) + 
  geom_line(size = 1.1, alpha = 0.8, show.legend = FALSE) + 
  scale_x_log10() +
  scale_y_log10()

rank_subset <- freq_by_rank %>% 
  filter(rank < 500,
         rank > 10)
lm(log10(`term frequency`) ~ log10(rank), data = rank_subset)

Call:
lm(formula = log10(`term frequency`) ~ log10(rank), data = rank_subset)

Coefficients:
(Intercept)  log10(rank)  
    -0.4922      -1.1332  

freq_by_rank %>% 
  ggplot(aes(rank, `term frequency`, color =album)) + 
  geom_abline(intercept = -0.62, slope = -1.1, color = "gray50", linetype = 2) +
  geom_line(size = 1.1, alpha = 0.8, show.legend = FALSE) + 
  scale_x_log10() +
  scale_y_log10()

lyric_words_4albums <- lyric_words_4albums %>%
  bind_tf_idf(word, album, n)
lyric_words_4albums

lyric_words_4albums %>%
  select(-total) %>%
  arrange(desc(tf_idf))

lyric_words_4albums %>%
  arrange(desc(tf_idf)) %>%
  mutate(word = factor(word, levels = rev(unique(word)))) %>% 
  group_by(album) %>% 
  top_n(15) %>% 
  ungroup %>%
  ggplot(aes(word, tf_idf, fill = album)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~album, ncol = 2, scales = "free") +
  coord_flip()

Selecting by tf_idf

3.4 A corpus of physics texts