Manifestos

1. This code reads the manifesto data.

manifestos <- read_excel("manifestos.xlsx")

manifestos

This code unnests the words in the manifestos, which separates the words into individual rows.

manifestos <- manifestos %>%
  unnest_tokens(word, text)

manifestos

2. This code finds the lexical diversity, or use of distinct words, the lexical density, which finds what proportion of words are distinct, and the number of words in each manifesto.

manifestos %>% 
  group_by(author) %>% 
  summarise(num_words = n(),
            lex_diversity = n_distinct(word), 
            lex_density = n_distinct(word)/n())

3. This code finds the mean word length for each manifesto and displays it in a table.

manifestos %>%
  group_by(author) %>% 
  mutate(word_length = nchar(word)) %>% 
  summarize(mean_word_length = mean(word_length)) %>% 
  arrange(-mean_word_length) 

4. This code finds the length of the words in each manifesto and creates graphs for each document.

manifestos %>%
  mutate(word_length = nchar(word)) %>% 
  ggplot(aes(word_length)) +
  geom_histogram(binwidth = 1) +
  facet_wrap(vars(author), scales = "free_y") +
  theme_minimal() +
  labs(x = "Word Lengths", y = "Words", title = "Manifesto Word Lengths by Author") 

5. This code removes the stop words, like “the”, “and”, “you”, from each manifesto so only the distinct words are counted.

manifestos %>%
  anti_join(stop_words)

Joining, by = "word"

This code finds the top five most common words in each manifesto, without the stop words, and creates little graphs to disply them.

manifestos %>%
  anti_join(stop_words) %>% 
  group_by(author) %>% 
  count(word, sort = T) %>%
  top_n(5) %>% 
  ungroup() %>% 
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "Most common words") +
  facet_wrap(vars(author), scales = "free") +
  scale_fill_viridis_d() +
  labs(title = "Most Common Words in Each Manifesto") +
  theme_minimal() +
  coord_flip()

Joining, by = "word"
Selecting by n

6. The code below counts and combines the words in each manifesto and the common words between them and uses that to calculate and display the tf-idfs.

manifesto_counts <- manifestos %>%
  unnest_tokens(word, text) %>%
  count(author, word, sort = TRUE)

total_words <- manifesto_counts %>%
  group_by(author) %>%
  summarize(total = sum(n))

manifesto_counts <- left_join(manifesto_counts, total_words)

Joining, by = "author"

manifesto_tf_idf <- manifesto_counts %>%
  bind_tf_idf(word, author, n)

manifesto_tf_idf %>%
  arrange(-tf_idf)

This code creates little graphs of the highest tf-idfs for each manifesto.

manifesto_tf_idf %>%
  arrange(-tf_idf) %>%
  mutate(word = factor(word, levels = rev(unique(word)))) %>% 
  group_by(author) %>% 
  top_n(5) %>% 
  ggplot(aes(word, tf_idf, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~author, scales = "free") +
  coord_flip() +
  theme_minimal() +
  scale_fill_viridis_d() +
  labs(title = "Most Distinctive Words in Each Manifesto")

Selecting by tf_idf