text_analysis

library(tidyverse)

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ ggplot2   3.4.4     ✔ tibble    3.2.1
✔ lubridate 1.9.3     ✔ tidyr     1.3.1
✔ purrr     1.0.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(tidytext)        # package for text analysis

Warning: package 'tidytext' was built under R version 4.3.3

library(readxl)          # reads excel files, the format I used for the data

manifestos <- read_excel("manifestos.xlsx")

Obtain several measures of complexity, including number of words, number of distinct words, the density, and create a graph of distributions of word lengths for each author.

manifestos_words <- manifestos |>
  unnest_tokens(word, text)

1a) Number of words, number of distinct words, and the density of distinct words

manifestos_words |> 
  group_by(author) |> 
  summarize(num_words = n(), 
            lex_diversity = n_distinct(word),
            lex_density = lex_diversity/num_words)

# A tibble: 6 × 4
  author       num_words lex_diversity lex_density
  <chr>            <int>         <int>       <dbl>
1 Auvinen           1750           624       0.357
2 Breivik           5186          1457       0.281
3 Cho               1971           608       0.308
4 HarperMercer      1589           580       0.365
5 Rodger            6216          1310       0.211
6 Unabomber         5100          1300       0.255

1a) The manifesto with the most words was authored by Rodger and the least was by HarperMercer. The manifesto with the most distinct words was authored by Breivik and the least was by HarperMercer. Lastly, the manifesto with the most distinct words when taken in correlation to the amount of words written was by HarperMercer and the least was by Rodger.

1b)Average Word Length

manifestos_words |> 
  group_by(author) |> 
  mutate(word_length = nchar(word)) |> 
  summarize(mean_word_length = mean(word_length)) |> 
  arrange(-mean_word_length)

# A tibble: 6 × 2
  author       mean_word_length
  <chr>                   <dbl>
1 Breivik                  5.15
2 Unabomber                4.91
3 Auvinen                  4.85
4 Cho                      4.25
5 Rodger                   4.15
6 HarperMercer             4.11

1b) The manifesto with the longest average word was authored by Breivik and the smallest average word was by HarperMercer.

1c) Word Length Distribution

manifestos_words |>
  mutate(word_length = nchar(word)) |> 
  ggplot(aes(word_length)) +
  facet_wrap(vars(author), scales = "free_y") +
  geom_histogram(binwidth = 1) +
  labs(title = "Word Length Distribution in Each Manifesto")

1c) The graphs show the distribution of the length of the words used by each author. All authors had their most significant distribution of words with words containing 2-4 letters.

Graph both the most common words used by each author, and the most common words with words removed.

manifestos_words |> 
  count(word, sort = T)

# A tibble: 3,863 × 2
   word      n
   <chr> <int>
 1 the    1189
 2 of      724
 3 and     617
 4 to      603
 5 a       455
 6 i       392
 7 in      354
 8 that    284
 9 my      269
10 is      238
# ℹ 3,853 more rows

2a) Graph of the most common words used by each author

manifestos_words |> 
  group_by(author) |>
  count(word, sort = T) |> 
  top_n(5) |>
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  facet_wrap(~author, scales = "free") +           # creates separate graphs for each author
  scale_fill_viridis_d() +                         # uses a nicer color scheme
  theme_minimal() +                                # removes the gray background
  labs(x = NULL, y = "Most common words")

Selecting by n

2a) For all manifestos, except for Cho’s, “the” was the most common word used and the only word to make all 6 graphs. The longest word to make any graph was “black” by Auvinen’s Manifesto.

stop_words

# A tibble: 1,149 × 2
   word        lexicon
   <chr>       <chr>  
 1 a           SMART  
 2 a's         SMART  
 3 able        SMART  
 4 about       SMART  
 5 above       SMART  
 6 according   SMART  
 7 accordingly SMART  
 8 across      SMART  
 9 actually    SMART  
10 after       SMART  
# ℹ 1,139 more rows

stop_words |> 
  group_by(lexicon) |> 
  count()

# A tibble: 3 × 2
# Groups:   lexicon [3]
  lexicon      n
  <chr>    <int>
1 SMART      571
2 onix       404
3 snowball   174

stop_words |> 
  filter(lexicon == "snowball") -> snowball

manifestos_words |>
  anti_join(snowball) |> 
  group_by(author) |> 
  count(word, sort = T)

Joining with `by = join_by(word)`

# A tibble: 5,314 × 3
# Groups:   author [6]
   author    word            n
   <chr>     <chr>       <int>
 1 Rodger    time           44
 2 Breivik   cultural       42
 3 Breivik   political      42
 4 Rodger    school         39
 5 Breivik   marxism        37
 6 Breivik   correctness    36
 7 Unabomber society        36
 8 Rodger    life           34
 9 Rodger    first          33
10 Rodger    house          31
# ℹ 5,304 more rows

2b) Graph of most common words with stop words removed

manifestos_words |>
  anti_join(snowball) |> 
  group_by(author) |> 
  count(word, sort = T) |> 
  top_n(5) |> 
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "Most common words") +
  facet_wrap(vars(author), scales = "free") +
  scale_fill_viridis_d() +
  theme_minimal() +
  coord_flip()

Joining with `by = join_by(word)`
Selecting by n

2b) After taking out the stop words, the diversity of language used by each manifesto is greatly enhanced. While the diversity has been enhanced, the amount of times a common word was used has greatly decreased.

Create both a table and a graph of TF-IDF

3a) Creata a table of TF-IDF

manifestos_word_counts <- manifestos |>             # This counts each word per author
  unnest_tokens(word, text) |>
  count(author, word, sort = TRUE) 

total_words <- manifestos_word_counts |>               # This counts total words per author
  group_by(author) |> 
  summarize(total = sum(n))

manifestos_word_counts <- left_join(manifestos_word_counts, total_words)    # Joins the two

Joining with `by = join_by(author)`

manifestos_tf_idf <- manifestos_word_counts |>             # Calculates tf-idf
  bind_tf_idf(word, author, n)

manifestos_tf_idf |>                                   # Displays it
  arrange(-tf_idf)

# A tibble: 5,879 × 7
   author    word            n total      tf   idf  tf_idf
   <chr>     <chr>       <int> <int>   <dbl> <dbl>   <dbl>
 1 Cho       you           130  1971 0.0660  0.405 0.0267 
 2 Cho       your           34  1971 0.0173  1.10  0.0190 
 3 Cho       fuck           19  1971 0.00964 1.79  0.0173 
 4 Breivik   cultural       42  5186 0.00810 1.79  0.0145 
 5 Breivik   marxism        37  5186 0.00713 1.79  0.0128 
 6 Auvinen   intelligent    10  1750 0.00571 1.79  0.0102 
 7 Auvinen   minded         10  1750 0.00571 1.79  0.0102 
 8 Cho       jesus          18  1971 0.00913 1.10  0.0100 
 9 Unabomber leftist        27  5100 0.00529 1.79  0.00949
10 Breivik   frankfurt      27  5186 0.00521 1.79  0.00933
# ℹ 5,869 more rows

3a) The top three words that were used most frequently in one manifesto and comparatively less so in others all came from Cho and were “you”, “your”, and “fuck”.

3b) Create a graph of TF-IDF

manifestos_tf_idf |>
  arrange(-tf_idf) |>
  mutate(word = factor(word, levels = rev(unique(word)))) |> 
  group_by(author) |> 
  top_n(5) |> 
  ggplot(aes(word, tf_idf, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~author, scales = "free") +
  theme_minimal() +
  scale_fill_viridis_d() +
  labs(title = "Most distinctive words in each manifesto") +
  coord_flip()

Selecting by tf_idf

3b) By graphing the most distinctive words for each author, we can see how each author wrote about different subjects. For example, Breivik was concerned with Marxist theory and the Unabomber was enamored by the left-wing of political matters.

Do a sentiment analysis with a graph of the most common positive and negative words

bing <- get_sentiments("bing")
bing

# A tibble: 6,786 × 2
   word        sentiment
   <chr>       <chr>    
 1 2-faces     negative 
 2 abnormal    negative 
 3 abolish     negative 
 4 abominable  negative 
 5 abominably  negative 
 6 abominate   negative 
 7 abomination negative 
 8 abort       negative 
 9 aborted     negative 
10 aborts      negative 
# ℹ 6,776 more rows

manifestos_words |> 
  inner_join(bing) |> 
  count(word, sentiment, sort = TRUE) |> 

  group_by(sentiment) |>
  top_n(10) |>
  ungroup() |>
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(vars(sentiment), scales = "free") +
  labs(y = "Suicide notes: Words that contribute the most to each sentiment",
       x = NULL) +
  scale_fill_viridis_d() +
  coord_flip() +
  theme_minimal()

Joining with `by = join_by(word)`
Selecting by n

This graph analyzes the amount of times a word with positive or negative sentiment was used. As the x-axis shows, positive sentiments were far more common then negative.
Create tables of the most common bigrams, and the most common bigrams after removing stop words.

5a) Create a table of the most common bigrams

manifestos |>
  unnest_tokens(bigram, text, token = "ngrams", n = 2) |> 
  select(bigram) -> manifestos_bigrams

manifestos_bigrams |> 
  count(bigram, sort = T)

# A tibble: 14,954 × 2
   bigram                    n
   <chr>                 <int>
 1 of the                  159
 2 in the                   81
 3 to the                   69
 4 i was                    60
 5 and the                  56
 6 to be                    43
 7 political correctness    37
 8 i had                    34
 9 it is                    34
10 it was                   30
# ℹ 14,944 more rows

5a) This table shows the most common bigrams, two words in a row, in all the manifestos. The most common bigram was “of the” and it was utilized almost twice as much as the next most used bigram of “in the”.

5b) Create a table of the most common bigrams after removing stop words.

manifestos_bigrams |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(!word1 %in% snowball$word) |>
  filter(!word2 %in% snowball$word) |> 
  unite(bigram, word1, word2, sep = " ") |> 
  count(bigram, sort = T)

# A tibble: 4,115 × 2
   bigram                    n
   <chr>                 <int>
 1 political correctness    37
 2 frankfurt school         22
 3 jesus christ             17
 4 cultural marxism         16
 5 black man                12
 6 human race               12
 7 modern leftism            9
 8 power process             9
 9 surrogate activities      9
10 will never                9
# ℹ 4,105 more rows

5b) After removing far more common words, the most common bigram is “political correctness” and the next most common is “frankfurt school”.

Select two or three “first words” of your own and create a graph showing the most common words that came after those first words.

first_word <- c("of", "in")                                  # these need to be lowercase

manifestos_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |>       # separate the two words
  filter(word1 %in% first_word) |>                          # find first words from our list
  count(word1, word2, wt = n, sort = TRUE) |> 

  mutate(word2 = factor(word2, levels = rev(unique(word2)))) |>     # put the words in order
  group_by(word1) |> 
  top_n(5) |> 
  ggplot(aes(word2, n, fill = word1)) +                          
  scale_fill_viridis_d() +                                           # set the color palette
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = NULL, title = "Word following:") +
  facet_wrap(~word1, scales = "free") +
  coord_flip() +
  theme_minimal()

Selecting by n

By choosing the top starters of the bigrams, it is clear that “the” as a follower to the first words will have appeared a great number of times. Other common second words between both first words were “my” and “a”.