Text_manifesto_Assignment

library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ ggplot2   3.5.0     ✔ tibble    3.2.1
✔ lubridate 1.9.3     ✔ tidyr     1.3.0
✔ purrr     1.0.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(tidytext)
library(readxl)

manifestos <- read_excel("manifestos.xlsx")

manifesto_words <- manifestos |>
  unnest_tokens(word, text)

manifesto_words |> 
  group_by(author) |> 
  summarize(num_words = n(),
            lex_diversity = n_distinct(word),
            lex_density = lex_diversity/num_words,)

# A tibble: 6 × 4
  author       num_words lex_diversity lex_density
  <chr>            <int>         <int>       <dbl>
1 Auvinen           1750           624       0.357
2 Breivik           5186          1457       0.281
3 Cho               1971           608       0.308
4 HarperMercer      1589           580       0.365
5 Rodger            6216          1310       0.211
6 Unabomber         5100          1300       0.255

This table shows the number of words, distinct words and density of words for each author.

manifesto_words |>
  mutate(word_length = nchar(word)) |> 
  ggplot(aes(word_length)) +
  facet_wrap(vars(author), scales = "free_y") +
  geom_histogram(binwidth = 1) +
  theme_minimal() +
  labs(title = "Word length distributions for each manifesto")

These graphs show the distribution of word lengths for each author.

manifesto_words |> 
  group_by(author) |>
  count(word, sort = T) |>
  top_n(5) |>
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  facet_wrap(~author, scales = "free") +
  scale_fill_viridis_d() +                        
  theme_minimal() +                               
  labs(x = NULL, y = "Most common words")

Selecting by n

The graphs above show the most common words used by each author.

stop_words |> 
  filter(lexicon == "snowball") -> snowball

manifesto_words |>
  anti_join(snowball) |>
  group_by(author) |>
  count(word, sort = T) |>
  top_n(5) |> 
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "Most common words") +
  facet_wrap(vars(author), scales = "free") +
  scale_fill_viridis_d() +
  theme_minimal() +
  coord_flip()

Joining with `by = join_by(word)`
Selecting by n

The graphs above show the most common words used by each author; not including stop words.

manifesto_word_counts <- manifestos |>
  unnest_tokens(word, text) |>
  count(author, word, sort = TRUE) 

total_words <- manifesto_word_counts |> 
  group_by(author) |> 
  summarize(total = sum(n))

manifesto_word_counts <- left_join(manifesto_word_counts, total_words)

Joining with `by = join_by(author)`

manifesto_tf_idf <- manifesto_word_counts |>
  bind_tf_idf(word, author, n)

manifesto_tf_idf |> 
  arrange(-tf_idf)

# A tibble: 5,879 × 7
   author    word            n total      tf   idf  tf_idf
   <chr>     <chr>       <int> <int>   <dbl> <dbl>   <dbl>
 1 Cho       you           130  1971 0.0660  0.405 0.0267 
 2 Cho       your           34  1971 0.0173  1.10  0.0190 
 3 Cho       fuck           19  1971 0.00964 1.79  0.0173 
 4 Breivik   cultural       42  5186 0.00810 1.79  0.0145 
 5 Breivik   marxism        37  5186 0.00713 1.79  0.0128 
 6 Auvinen   intelligent    10  1750 0.00571 1.79  0.0102 
 7 Auvinen   minded         10  1750 0.00571 1.79  0.0102 
 8 Cho       jesus          18  1971 0.00913 1.10  0.0100 
 9 Unabomber leftist        27  5100 0.00529 1.79  0.00949
10 Breivik   frankfurt      27  5186 0.00521 1.79  0.00933
# ℹ 5,869 more rows

This table shows the frequency of each term that most differentiates each document.

manifesto_tf_idf |>
  arrange(-tf_idf) |>
  mutate(word = factor(word, levels = rev(unique(word)))) |> 
  group_by(author) |> 
  top_n(5) |> 
  ggplot(aes(word, tf_idf, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~author, scales = "free") +
  theme_minimal() +
  scale_fill_viridis_d() +
  labs(title = "Most distinctive words in each manifesto") +
  coord_flip()

Selecting by tf_idf

These graphs show the frequency of each term that most differentiates each document, otherwise known as the TF-IDF.

bing <- get_sentiments("bing")
bing

# A tibble: 6,786 × 2
   word        sentiment
   <chr>       <chr>    
 1 2-faces     negative 
 2 abnormal    negative 
 3 abolish     negative 
 4 abominable  negative 
 5 abominably  negative 
 6 abominate   negative 
 7 abomination negative 
 8 abort       negative 
 9 aborted     negative 
10 aborts      negative 
# ℹ 6,776 more rows

manifesto_words |> 
  inner_join(bing) |> 
  count(word, sentiment, sort = TRUE) |>

  group_by(sentiment) |>
  top_n(10) |>
  ungroup() |>
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(vars(sentiment), scales = "free") +
  labs(y = "Manifestos: Words that contribute the most to each sentiment",
       x = NULL) +
  scale_fill_viridis_d() +
  coord_flip() +
  theme_minimal()

Joining with `by = join_by(word)`
Selecting by n

These graphs show the most common positive and negative words in the manifestos.

manifestos |>
  unnest_tokens(bigram, text, token = "ngrams", n = 2) |> 
  select(bigram) ->  manifesto_bigrams

manifesto_bigrams |>
    count(bigram, sort = T)

# A tibble: 14,954 × 2
   bigram                    n
   <chr>                 <int>
 1 of the                  159
 2 in the                   81
 3 to the                   69
 4 i was                    60
 5 and the                  56
 6 to be                    43
 7 political correctness    37
 8 i had                    34
 9 it is                    34
10 it was                   30
# ℹ 14,944 more rows

This table shows the most common bigrams present in the manifestos.

manifesto_bigrams |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(!word1 %in% snowball$word) |>
  filter(!word2 %in% snowball$word) |> 
  unite(bigram, word1, word2, sep = " ")

# A tibble: 4,861 × 1
   bigram               
   <chr>                
 1 industrial society   
 2 future introduction  
 3 introduction 1       
 4 industrial revolution
 5 human race           
 6 greatly increased    
 7 life expectancy      
 8 advanced countries   
 9 destabilized society 
10 made life            
# ℹ 4,851 more rows

This table shows the most common bigrams present in the manifestos when the stop words are removed.

first_word <- c("my", "world")
manifesto_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(word1 %in% first_word) |>
  count(word1, word2, wt = n, sort = TRUE)

# A tibble: 142 × 3
   word1 word2          n
   <chr> <chr>      <int>
 1 my    life          28
 2 my    mother        25
 3 my    father        21
 4 my    parents       18
 5 my    first         10
 6 my    father’s       7
 7 my    favorite       7
 8 my    new            6
 9 world population     6
10 my    class          5
# ℹ 132 more rows

This table shows the the most common words that came after the words “my” and “world” in the manifestos.

first_word <- c("my", "world")
manifesto_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |>    
  filter(word1 %in% first_word) |> 
  count(word1, word2, wt = n, sort = TRUE) |>
  mutate(word2 = factor(word2, levels = rev(unique(word2)))) |> 
  group_by(word1) |>
  top_n(5) |>
  ggplot(aes(word2, n, fill = word1)) +                          
  scale_fill_viridis_d() +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = NULL, title = "Word following:") +
  facet_wrap(~word1, scales = "free") +
  coord_flip() +
  theme_minimal()

Selecting by n

These graphs show the most common words that came after the words “my” or “world” in the manifestos.