thesisdataV1

library(tidyverse)
library(billboard)
library(dplyr)
library(tidytext)
library(readxl)
library(wordcloud2)

added_music <- read_excel("thesisDataFinal.xlsx")

Code for generating datasets

added_music %>% 
  unnest_tokens(word, Lyrics) -> teen_words

wordclouds

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook")) %>% 
  count(word, sort = TRUE) %>% 
  wordcloud2()

## Joining, by = "word"

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook")) %>% 
  count(word, sort = TRUE) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word      n
##    <chr> <int>
##  1 love  15904
##  2 yeah  11250
##  3 baby  10736
##  4 time   5800
##  5 wanna  5782
##  6 girl   5457
##  7 gonna  4566
##  8 ooh    4428
##  9 feel   4184
## 10 night  4156

Word cloud filtered by ISO yes.

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(ISO == "Yes") %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook")) %>% 
  count(word, sort = TRUE) %>% 
  wordcloud2()

## Joining, by = "word"

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(ISO == "Yes") %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook")) %>% 
  count(word, sort = TRUE) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word      n
##    <chr> <int>
##  1 love  10136
##  2 baby   9435
##  3 yeah   8339
##  4 girl   5374
##  5 wanna  4403
##  6 ooh    3166
##  7 time   3121
##  8 gonna  2729
##  9 night  2668
## 10 feel   2517

Word cloud for ISO no.

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(ISO == "No") %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook", "love")) %>% 
  count(word, sort = TRUE) %>% 
  wordcloud2()

## Joining, by = "word"

teen_words %>% 
  anti_join(stop_words) %>% 
  filter(ISO == "No") %>% 
  filter(!word %in% c("verse", "chorus", "1", "2", "bridge", "hook", "love")) %>% 
  count(word, sort = TRUE) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word      n
##    <chr> <int>
##  1 yeah   2911
##  2 time   2677
##  3 gonna  1837
##  4 feel   1666
##  5 life   1592
##  6 heart  1512
##  7 night  1488
##  8 wanna  1379
##  9 baby   1301
## 10 day    1268

Gender Distribution

This gives me gender distribution by ISO. Add second graph that evaluates this by proportion

added_music %>% 
  group_by(ISO) %>% 
  count(Gender) -> genderISO

genderISO %>% 
  ggplot(aes(Gender, n, fill = ISO)) + geom_bar(stat = "identity") + theme_classic() +
   labs(x = "Artist Gender",
       y = "Number of Songs",
       title = "Number of ISO across gender for all years")

ggplot(genderISO) +
  geom_col(aes(x = Gender, y = n, fill = ISO), position = "fill") + theme_classic() +
  labs(x = "Artist Gender",
       y = "Proportion",
       title = "Proportion of ISO across gender for all years")

Filtering and comparing by gender

added_music %>% 
  filter(ISO == "Yes") %>% 
  group_by(Year) %>% 
  count(Gender) %>% 
  ggplot(aes(Year, n, color = Gender)) + geom_line() +  theme_classic() +
  labs(x = "Year",
       y = "Number",
       title = "Trend Over 50 Years of Yes ISO")

added_music %>% 
  filter(ISO == "No") %>% 
  group_by(Year) %>% 
  count(Gender) %>% 
  ggplot(aes(Year, n, color = Gender)) + geom_line() +  theme_classic() +
  labs(x = "Year",
       y = "Number",
       title = "Trend Over 50 Years of No ISO")

ISO

this graph gives me ISO over the years

added_music %>% 
  group_by(Year) %>% 
  count(ISO) ->trial

trial %>% 
  ggplot(aes(Year, n, color = ISO)) + geom_line()

genre over time

added_music %>% 
  group_by(Year) %>% 
  count(Genre)->trial2

trial2 %>% 
  filter(Genre %in% c("Pop", "Rock", "R&B", "Rap", "Hip Hop", "Country", "Soul", "Pop Rock",
                      "Dance Pop", "Alt Rock")) %>% 
  ggplot(aes(Year, n, color = Genre)) + geom_line() + theme_classic()  + 
  labs(x = "Year",
       y = "Number of Songs",
       title = "Number of songs by genre over 50 years")

genre filtered for ISO over time

added_music %>% 
    filter(ISO == "Yes") %>% 
  group_by(Year) %>% 
  count(Genre) %>% 
  filter(Genre %in% c("Pop", "Rock", "R&B", "Rap", "Hip Hop", "Country", "Soul", "Pop Rock",
                      "Dance Pop", "Alt Rock")) %>% 
  ggplot(aes(Year, n, color = Genre)) + geom_line() + theme_classic() +
  labs(x = "Year",
       y = "Number of Songs",
       title = "Number of songs by genre over 50 years",
       subtitle = "Filtered for ISO present")

Words

This should give me frequency of list over time

explicit <- c("pussy", "ass", "baby", "girl", "body", "dick", "17", "seventeen", "sex", "sexy",
              "sexiness", "16", "sixteen", "18", "eighteen", "nineteen", "19", "15", "fifteen",
              "teen", "teenager", "teens", "teenagers", "booty", "cock", "cocks", "cum", "hoe",
              "hoes", "stripper", "strippers", "whore", "whores", "foreplay", "butt", "butts", 
              "girls", "girlies", "tits", "titties", "horny", "virgin", "virgins", "teenage")

explicit2 <- c("pussy", "ass", "dick", "17", "seventeen", "sex", "sexy",
              "sexiness", "16", "sixteen", "18", "eighteen", "nineteen", "19", "15", "fifteen",
              "teen", "teenager", "teens", "teenagers", "booty", "cock", "cocks", "cum", "hoe",
              "hoes", "stripper", "strippers", "whore", "whores", "foreplay", "butt", "butts", 
              "girls", "girlies", "tits", "titties", "horny", "virgin", "virgins", "teenage")

age <- c("17", "seventeen","16", "sixteen", "18", "eighteen", "nineteen", "19", "15", "fifteen",
              "teen", "teenager", "teens", "teenagers", "teenage")

body <- c("pussy", "ass", "dick", "booty", "cock", "cocks", "cum", "butt", "butts", "tits",
          "titties")

sex <- c("sex", "sexy", "sexiness", "hoe", "hoes", "stripper", "strippers", "whore", "whores",
         "foreplay", "horny", "virgin", "virgins")

the number of times that these words show up in ISO songs, not the number of songs that have them

teen_words %>% 
  filter(ISO == "Yes") -> ISOfilter


ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% explicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% explicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_point() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% explicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word      n
##    <chr> <int>
##  1 baby   9435
##  2 girl   5374
##  3 body   1503
##  4 girls   955
##  5 ass     861
##  6 sexy    462
##  7 hoes    352
##  8 pussy   313
##  9 booty   304
## 10 dick    242

rerun without baby, girl, and body

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% explicit2) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, removing baby, girl, and body")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% explicit2) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_point() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% explicit2) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word      n
##    <chr> <int>
##  1 girls   955
##  2 ass     861
##  3 sexy    462
##  4 hoes    352
##  5 pussy   313
##  6 booty   304
##  7 dick    242
##  8 sex     240
##  9 hoe     145
## 10 butt     89

age

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% age) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, for age words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% age) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_point() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, age words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% age) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word          n
##    <chr>     <int>
##  1 seventeen    31
##  2 sixteen      30
##  3 18           27
##  4 teenage      25
##  5 15           21
##  6 17           19
##  7 fifteen      12
##  8 eighteen      8
##  9 16            5
## 10 teen          3

body

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% body) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, for body words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% body) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_point() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, body words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% body) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word        n
##    <chr>   <int>
##  1 ass       861
##  2 pussy     313
##  3 booty     304
##  4 dick      242
##  5 butt       89
##  6 titties    44
##  7 cock       34
##  8 cum        33
##  9 tits       18
## 10 butts       5

sex words

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% sex) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, for sex words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% sex) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_line() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of explicit words over 50 years",
       subtitle = "Filtered by ISO present, sex words")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% sex) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word         n
##    <chr>    <int>
##  1 sexy       462
##  2 hoes       352
##  3 sex        240
##  4 hoe        145
##  5 horny       35
##  6 virgin      23
##  7 stripper    18
##  8 whores      18
##  9 whore       14
## 10 foreplay    11

Implicit lexicon

implicit <- c("deep", "bed", "beds", "thick", "lollipop", "lollipops", "dirty", "shake", "shaking", "kiss",
              "shorty", "shawty", "shortie", "night", "nights", "bounce", "chick", "chicks",
              "hip", "hips", "fucking", "fuck", "smack", "school", "freak", "freaky", "twerk",
              "twerking", "mama", "kisses", "kissing", "mamas")

teen_words %>% 
  filter(ISO == "Yes") -> ISOfilter


ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% implicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(x = reorder(word, -n), y = n, fill = word)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
       title = "Frequency of implicit words over 50 years",
       subtitle = "Filtered by ISO present")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  group_by(Year) %>% 
  filter(word %in% implicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  ggplot(aes(Year, n, color = word)) + geom_point() +
  theme_classic() +
  labs(x = "Lyric",
       y = "Frequency of lyric",
title = "Frequency of implicit words over 50 years",
       subtitle = "Filtered by ISO present")

## Joining, by = "word"

ISOfilter %>% 
  anti_join(stop_words) %>% 
  filter(word %in% implicit) %>% 
  count(word, sort = TRUE) %>% 
  arrange(desc(n)) %>% 
  head(10)

## Joining, by = "word"

## # A tibble: 10 × 2
##    word       n
##    <chr>  <int>
##  1 night   2668
##  2 fuck    1146
##  3 shake    879
##  4 kiss     785
##  5 mama     494
##  6 deep     422
##  7 shawty   332
##  8 bed      328
##  9 chick    273
## 10 shorty   266

Top artists

literal_songs <- teen_words %>%
  filter(word %in% explicit) %>%
  group_by(Song)

literal_songs %>%
  group_by(Artist) %>%
  summarize(n_distinct(Song)) -> redone

redone %>%
  arrange(desc(`n_distinct(Song)`)) %>%
  head(10) %>%
  ggplot(aes(Artist, `n_distinct(Song)`, fill = Artist)) + geom_bar(stat = "identity") +
  theme_classic() +
  labs(x = "Artist",
       y = "Number of Unique Songs",
       title = "Number of Unique Songs by Artist With Words in Literal Lexicon",
       subtitle = "Each song contains one or more of the literal lexicon words") +
  scale_x_discrete(guide = guide_axis(n.dodge = 2))

Running a significance test

Gender

gen_data = data.frame(added_music$Gender, added_music$ISO)

gen_data = table(added_music$Gender, added_music$ISO)

print(gen_data)

##            
##               No  yes  Yes
##   Female     609    0  705
##   Male      1363    1 2030
##   Mixed      192    0  183
##   Nonbinary    8    0    1
##   Unknown      7    0    1

print(chisq.test(gen_data))

## Warning in chisq.test(gen_data): Chi-squared approximation may be incorrect

## 
##  Pearson's Chi-squared test
## 
## data:  gen_data
## X-squared = 41.994, df = 8, p-value = 1.358e-06

Featured

gen_data2 = data.frame(added_music$Featured, added_music$ISO)

gen_data2 = table(added_music$Featured, added_music$ISO)

print(gen_data2)

##      
##         No  yes  Yes
##   No  1974    1 2196
##   Yes  205    0  724

print(chisq.test(gen_data2))

## Warning in chisq.test(gen_data2): Chi-squared approximation may be incorrect

## 
##  Pearson's Chi-squared test
## 
## data:  gen_data2
## X-squared = 198.52, df = 2, p-value < 2.2e-16

Band

gen_data3 = data.frame(added_music$Band, added_music$ISO)

gen_data3 = table(added_music$Band, added_music$ISO)

print(gen_data3)

##      
##         No  yes  Yes
##   no     0    0    1
##   No  1270    0 1980
##   Yes  909    1  939

print(chisq.test(gen_data3))

## Warning in chisq.test(gen_data3): Chi-squared approximation may be incorrect

## 
##  Pearson's Chi-squared test
## 
## data:  gen_data3
## X-squared = 51.717, df = 4, p-value = 1.581e-10

bigram analysis

added_music %>% 
  unnest_tokens(bigram, Lyrics, token="ngrams", n=2) -> d_bigrams

 d_bigrams %>% 
   separate(bigram, c("word1", "word2"), sep = " ") -> d_bigrams
 
 d_bigrams %>%  
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word) %>% 
  filter(!word1 %in% NA) %>% 
  filter(!word2 %in% NA) -> d_bigrams
 
 d_bigrams %>% 
count(word1, word2, sort = TRUE) 

## # A tibble: 85,060 × 3
##    word1 word2      n
##    <chr> <chr>  <int>
##  1 yeah  yeah    3012
##  2 la    la      2497
##  3 na    na      2346
##  4 verse 1       2101
##  5 verse 2       2060
##  6 ooh   ooh     1413
##  7 pre   chorus  1000
##  8 da    da       770
##  9 verse 3        721
## 10 hey   hey      715
## # … with 85,050 more rows

added_music %>% 
  unnest_tokens(bigram, Lyrics, token="ngrams", n=2) -> d_bigrams2

 d_bigrams2 %>% 
   separate(bigram, c("word1", "word2"), sep = " ") -> d_bigrams2
 
 d_bigrams2 %>%  
   filter(ISO == "Yes") %>% 
  filter(!word1 %in% stop_words$word) %>%
  filter(!word2 %in% stop_words$word) %>% 
  filter(!word1 %in% NA) %>% 
  filter(!word2 %in% NA) -> d_bigrams2
 
 d_bigrams2 %>% 
count(word1, word2, sort = TRUE) 

## # A tibble: 62,708 × 3
##    word1 word2      n
##    <chr> <chr>  <int>
##  1 yeah  yeah    2229
##  2 la    la      1593
##  3 na    na      1404
##  4 verse 1       1130
##  5 verse 2       1114
##  6 ooh   ooh      978
##  7 baby  baby     608
##  8 da    da       600
##  9 pre   chorus   578
## 10 hey   hey      530
## # … with 62,698 more rows