text analysis assignment

library(tidyverse)

Warning: package 'ggplot2' was built under R version 4.3.3

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ ggplot2   3.5.0     ✔ tibble    3.2.1
✔ lubridate 1.9.3     ✔ tidyr     1.3.1
✔ purrr     1.0.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(tidytext)

Warning: package 'tidytext' was built under R version 4.3.3

library(readxl)

manifestos <- read_excel("manifestos.xlsx")

manifesto_words <- manifestos |>
  unnest_tokens(word, text)

bing <- get_sentiments("bing")
bing

# A tibble: 6,786 × 2
   word        sentiment
   <chr>       <chr>    
 1 2-faces     negative 
 2 abnormal    negative 
 3 abolish     negative 
 4 abominable  negative 
 5 abominably  negative 
 6 abominate   negative 
 7 abomination negative 
 8 abort       negative 
 9 aborted     negative 
10 aborts      negative 
# ℹ 6,776 more rows

manifestos |>
  unnest_tokens(bigram, text, token = "ngrams", n = 2) |> 
  select(bigram) -> manifesto_bigrams

first_word <- c("i", "you") 
manifesto_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(word1 %in% first_word) |> 
  count(word1, word2, wt = n, sort = TRUE)

# A tibble: 206 × 3
   word1 word2     n
   <chr> <chr> <int>
 1 i     was      60
 2 i     had      34
 3 you   have     22
 4 i     would    19
 5 i     have     18
 6 you   will     13
 7 i     am       11
 8 i     saw      10
 9 i     can       9
10 i     don’t     8
# ℹ 196 more rows

stop_words |> 
  filter(lexicon == "snowball") -> snowball

the following graph displays the proportion of words used based on the number of characters within each word.

manifesto_words |>
  mutate(word_length = nchar(word)) |> 
  ggplot(aes(word_length)) +
  facet_wrap(vars(author), scales = "free_y") +
  labs(title = "word length distributions by author") +
  geom_histogram(binwidth = 1)

The following displays most common words used by each author.

manifesto_words |> 
  group_by(author) |>
  count(word, sort = T) |>
  top_n(5) |>
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  facet_wrap(~author, scales = "free") +
  scale_fill_viridis_d() +
  theme_minimal() +                                
  labs(x = NULL, y = "Most common words")

Selecting by n

The following graph displays the most common words used by each author, excluding words that are extremely common such as “i” or “you”.

manifesto_words |>
  anti_join(snowball) |>
  group_by(author) |>
  count(word, sort = T) |>
  top_n(5) |> 
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "Most common words") +
  facet_wrap(vars(author), scales = "free") +
  scale_fill_viridis_d() +
  theme_minimal() +
  coord_flip()

Joining with `by = join_by(word)`
Selecting by n

The following table displays words that were unique to each author.

manifesto_word_counts <- manifestos |> 
  unnest_tokens(word, text) |>
  count(author, word, sort = TRUE) 

total_words <- manifesto_word_counts |> 
  group_by(author) |> 
  summarize(total = sum(n))

manifesto_word_counts <- left_join(manifesto_word_counts, total_words)

Joining with `by = join_by(author)`

manifesto_tf_idf <- manifesto_word_counts |>          
  bind_tf_idf(word, author, n)

manifesto_tf_idf |>                                 
  arrange(-tf_idf)

# A tibble: 5,879 × 7
   author    word            n total      tf   idf  tf_idf
   <chr>     <chr>       <int> <int>   <dbl> <dbl>   <dbl>
 1 Cho       you           130  1971 0.0660  0.405 0.0267 
 2 Cho       your           34  1971 0.0173  1.10  0.0190 
 3 Cho       fuck           19  1971 0.00964 1.79  0.0173 
 4 Breivik   cultural       42  5186 0.00810 1.79  0.0145 
 5 Breivik   marxism        37  5186 0.00713 1.79  0.0128 
 6 Auvinen   intelligent    10  1750 0.00571 1.79  0.0102 
 7 Auvinen   minded         10  1750 0.00571 1.79  0.0102 
 8 Cho       jesus          18  1971 0.00913 1.10  0.0100 
 9 Unabomber leftist        27  5100 0.00529 1.79  0.00949
10 Breivik   frankfurt      27  5186 0.00521 1.79  0.00933
# ℹ 5,869 more rows

The folllowing graph displays the words used by each author that were most unique to the manifesto they wrote.

manifesto_tf_idf |>
  arrange(-tf_idf) |>
  mutate(word = factor(word, levels = rev(unique(word)))) |> 
  group_by(author) |> 
  top_n(5) |> 
  ggplot(aes(word, tf_idf, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~author, scales = "free") +
  theme_minimal() +
  scale_fill_viridis_d() +
  labs(title = "Most distinctive words in each manifesto") +
  coord_flip()

Selecting by tf_idf

The following graph displays the most common words used by each author, designated by positive or negative connotations.

manifesto_words |> 
  inner_join(bing) |> 
  count(word, sentiment, sort = TRUE) |>
  group_by(sentiment) |>
  top_n(10) |>
  ungroup() |>
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(vars(sentiment), scales = "free") +
  labs(y = "manifestos: Words that contribute the most to each sentiment",
       x = NULL) +
  scale_fill_viridis_d() +
  coord_flip() +
  theme_minimal()

Joining with `by = join_by(word)`
Selecting by n

The following table displays the most common bigrams.

manifesto_bigrams |>
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  unite(bigram, word1, word2, sep = " ") |>
   count(bigram, sort = T)

# A tibble: 14,954 × 2
   bigram                    n
   <chr>                 <int>
 1 of the                  159
 2 in the                   81
 3 to the                   69
 4 i was                    60
 5 and the                  56
 6 to be                    43
 7 political correctness    37
 8 i had                    34
 9 it is                    34
10 it was                   30
# ℹ 14,944 more rows

The following table displays the most common bigrams, but removes words that are overly common.

manifesto_bigrams |>
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(!word1 %in% snowball$word) |>
  filter(!word2 %in% snowball$word) |>
  unite(bigram, word1, word2, sep = " ") |>
   count(bigram, sort = T)

# A tibble: 4,115 × 2
   bigram                    n
   <chr>                 <int>
 1 political correctness    37
 2 frankfurt school         22
 3 jesus christ             17
 4 cultural marxism         16
 5 black man                12
 6 human race               12
 7 modern leftism            9
 8 power process             9
 9 surrogate activities      9
10 will never                9
# ℹ 4,105 more rows

The following graph displays the most common words following “culture” and “society”.

first_word <- c("culture", "society") 
manifesto_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(word1 %in% first_word) |> 
  count(word1, word2, wt = n, sort = TRUE) |>
mutate(word2 = factor(word2, levels = rev(unique(word2)))) |>      
  group_by(word1) |> 
  top_n(5) |> 
  ggplot(aes(word2, n, fill = word1)) +                          
  scale_fill_viridis_d() +                                      
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = NULL, title = "Word following:") +
  facet_wrap(~word1, scales = "free") +
  coord_flip() +
  theme_minimal()

Selecting by n