text-analysis-finished

library(tidyverse)

Warning: package 'tidyverse' was built under R version 4.2.3

Warning: package 'ggplot2' was built under R version 4.2.3

Warning: package 'tibble' was built under R version 4.2.3

Warning: package 'tidyr' was built under R version 4.2.3

Warning: package 'readr' was built under R version 4.2.3

Warning: package 'purrr' was built under R version 4.2.3

Warning: package 'dplyr' was built under R version 4.2.3

Warning: package 'stringr' was built under R version 4.2.3

Warning: package 'forcats' was built under R version 4.2.3

Warning: package 'lubridate' was built under R version 4.2.3

── Attaching core tidyverse packages ──────────────────────── tidyverse 2.0.0 ──
✔ dplyr     1.1.4     ✔ readr     2.1.5
✔ forcats   1.0.0     ✔ stringr   1.5.1
✔ ggplot2   3.4.4     ✔ tibble    3.2.1
✔ lubridate 1.9.3     ✔ tidyr     1.3.1
✔ purrr     1.0.2     
── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──
✖ dplyr::filter() masks stats::filter()
✖ dplyr::lag()    masks stats::lag()
ℹ Use the conflicted package (<http://conflicted.r-lib.org/>) to force all conflicts to become errors

library(tidytext)        # package for text analysis

Warning: package 'tidytext' was built under R version 4.2.3

library(readxl)          # reads excel files, the format I used for the data

Warning: package 'readxl' was built under R version 4.2.3

massmurder_notes <- read_excel("manifestos.xlsx")

massmurder_words <- massmurder_notes |>
  unnest_tokens(word, text)

massmurder_words |> 
  group_by(author) |> 
  summarize(num_words = n(), lex_diversity = n_distinct(word))

# A tibble: 6 × 3
  author       num_words lex_diversity
  <chr>            <int>         <int>
1 Auvinen           1750           624
2 Breivik           5186          1457
3 Cho               1971           608
4 HarperMercer      1589           580
5 Rodger            6216          1310
6 Unabomber         5100          1300

A table of the total number of words used and the number of distinct words used by each author.

massmurder_words |>
  mutate(word_length = nchar(word)) |> 
  distinct(word, word_length, author) |> 
  arrange(-word_length)

# A tibble: 5,879 × 3
   word                word_length author   
   <chr>                     <int> <chr>    
 1 deindividualization          19 Auvinen  
 2 disproportionately           18 Breivik  
 3 oversocialization            17 Unabomber
 4 naturalselector89            17 Auvinen  
 5 hypersensitivity             16 Unabomber
 6 enthusiastically             16 Unabomber
 7 responsibilities             16 Cho      
 8 multiculturalism             16 Breivik  
 9 anthropologists              15 Unabomber
10 individualistic              15 Unabomber
# ℹ 5,869 more rows

A table showing the longest words used in the writings, how frequently they were used, and by which author.

massmurder_words |>
  mutate(word_length = nchar(word)) |> 
  ggplot(aes(word_length)) +
  facet_wrap(vars(author), scales = "free_y") +
  geom_histogram(binwidth = 1) +
  labs(title = "Word Lengths Of Mass Murderer Writings")+
  theme_minimal()

A set of graphs for each author of their word lengths. The graph shows an average word length for each author and how frequently that length showed up in the writing.

massmurder_words |> 
  group_by(author) |>
  count(word, sort = T) %>% 
  top_n(5) |>
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  coord_flip() +
  facet_wrap(~author, scales = "free") +           
  scale_fill_viridis_d() +                         
  theme_minimal() +                                
  labs(x = NULL, y = "Most common words")

Selecting by n

Graphs of the most common words used by each author.

massmurder_words |>
  anti_join(stop_words) %>% 
  group_by(author) %>% 
  count(word, sort = T) %>% 
  top_n(5) |> 
  ungroup() |> 
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "Most common words") +
  facet_wrap(vars(author), scales = "free") +
  scale_fill_viridis_d() +
  theme_minimal() +
  coord_flip()

Joining with `by = join_by(word)`
Selecting by n

Graphs of the most common words used by each author, but without “stop” words: the, and, of, a, to, I, etc.

murder_word_counts <- massmurder_notes |> 
  unnest_tokens(word, text) |>
  count(author, word, sort = TRUE) 

total_words <- murder_word_counts |>   
  group_by(author) |> 
  summarize(total = sum(n))

murder_word_counts <- left_join(murder_word_counts, total_words)

Joining with `by = join_by(author)`

murder_tf_idf <- murder_word_counts |>             
  bind_tf_idf(word, author, n)

murder_tf_idf |>                                   
  arrange(-tf_idf)

# A tibble: 5,879 × 7
   author    word            n total      tf   idf  tf_idf
   <chr>     <chr>       <int> <int>   <dbl> <dbl>   <dbl>
 1 Cho       you           130  1971 0.0660  0.405 0.0267 
 2 Cho       your           34  1971 0.0173  1.10  0.0190 
 3 Cho       fuck           19  1971 0.00964 1.79  0.0173 
 4 Breivik   cultural       42  5186 0.00810 1.79  0.0145 
 5 Breivik   marxism        37  5186 0.00713 1.79  0.0128 
 6 Auvinen   intelligent    10  1750 0.00571 1.79  0.0102 
 7 Auvinen   minded         10  1750 0.00571 1.79  0.0102 
 8 Cho       jesus          18  1971 0.00913 1.10  0.0100 
 9 Unabomber leftist        27  5100 0.00529 1.79  0.00949
10 Breivik   frankfurt      27  5186 0.00521 1.79  0.00933
# ℹ 5,869 more rows

Term-Frequency (tf)-Inverse-Document Frequency (idf) shows the relationship between a word in one writing compared to all the other writings. The table shows the words that are unique to one writing, the frequency of the word, and which author used said word.

murder_tf_idf |>
  arrange(-tf_idf) |>
  mutate(word = factor(word, levels = rev(unique(word)))) |> 
  group_by(author) |> 
  top_n(5) |> 
  ggplot(aes(word, tf_idf, fill = author)) +
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = "tf-idf") +
  facet_wrap(~author, scales = "free") +
  theme_minimal()+
  scale_fill_viridis_d()+
  labs(title = "Most distinctive words in each murder writing")+
  coord_flip()

Selecting by tf_idf

Graphs for each author for the tf-idf relationship.

bing <- get_sentiments("bing")
bing

# A tibble: 6,786 × 2
   word        sentiment
   <chr>       <chr>    
 1 2-faces     negative 
 2 abnormal    negative 
 3 abolish     negative 
 4 abominable  negative 
 5 abominably  negative 
 6 abominate   negative 
 7 abomination negative 
 8 abort       negative 
 9 aborted     negative 
10 aborts      negative 
# ℹ 6,776 more rows

massmurder_words |> 
  inner_join(bing) |> 
  count(word, sentiment, sort = TRUE) %>% 
  group_by(sentiment) |>
  top_n(10) |>
  ungroup() |>
  mutate(word = reorder(word, n)) |>
  ggplot(aes(word, n, fill = sentiment)) +
  geom_col(show.legend = FALSE) +
  facet_wrap(vars(sentiment), scales = "free") +
  labs(y = "Murder writings: Words that contribute the most to each",
       x = NULL) +
  scale_fill_viridis_d() +
  coord_flip() +
  theme_minimal()

Joining with `by = join_by(word)`
Selecting by n

Graphs of a sentiment analysis, taking words from the “bing” dictionary, that show the most common positive and negative words used in all of the writings.

massmurder_notes |>
  unnest_tokens(bigram, text, token = "ngrams", n = 2) |> 
  select(bigram) -> murder_bigrams

murder_bigrams |> 
  separate(bigram, c("word1", "word2"), sep = " ") |> 
  filter(!word1 %in% stop_words$word) |>
  filter(!word2 %in% stop_words$word) |> 
  unite(bigram, word1, word2, sep = " ") %>% 
  count(bigram, sort = T)

# A tibble: 2,448 × 2
   bigram                    n
   <chr>                 <int>
 1 political correctness    37
 2 frankfurt school         22
 3 jesus christ             17
 4 cultural marxism         16
 5 human race               12
 6 modern leftism            9
 7 power process             9
 8 surrogate activities      9
 9 ah mah                    7
10 western europe            7
# ℹ 2,438 more rows

The most common bigrams in all of the writings, and how often they are used (n).

first_word <- c("people", "world")                        
murder_bigrams |> 
  count(bigram, sort = T) |> 
  separate(bigram, c("word1", "word2"), sep = " ") |>      
  filter(word1 %in% first_word) |>                          
  count(word1, word2, wt = n, sort = TRUE) %>% 
    mutate(word2 = factor(word2, levels = rev(unique(word2)))) |>     
  group_by(word1) |> 
  top_n(5) |> 
  ggplot(aes(word2, n, fill = word1)) +                          
  scale_fill_viridis_d() +                                          
  geom_col(show.legend = FALSE) +
  labs(x = NULL, y = NULL, title = "Word following:") +
  facet_wrap(~word1, scales = "free") +
  coord_flip() +
  theme_minimal()

Selecting by n

Words that follow the selected words: people and world, and how frequently they are used as bigrams in all of the writings.