R Notebook

library(dplyr)
library(ggplot2)
setwd("C:/Users/amode/OneDrive/Documents/Dida 325")
data <- read.csv("final/data.csv")


popularity_range <- range(data$popularity)

print(popularity_range)

[1]   0 100

data$decade <- cut(data$year, breaks = seq(1920, 2020, by = 10),
                   labels = c("1920s", "1930s", "1940s", "1950s", "1960s", 
                              "1970s", "1980s", "1990s", "2000s", "2010s"))

data2 <- data %>%
  group_by(year) %>%
  filter(rank(desc(popularity)) <= n() * 0.1) %>%
  ungroup()
#https://stackoverflow.com/questions/48446816/explain-ungroup-in-dplyr

summary(data2)

      id                name             artists           duration_ms      release_date            year     
 Length:16946       Length:16946       Length:16946       Min.   :  30241   Length:16946       Min.   :1921  
 Class :character   Class :character   Class :character   1st Qu.: 176533   Class :character   1st Qu.:1957  
 Mode  :character   Mode  :character   Mode  :character   Median : 212640   Mode  :character   Median :1978  
                                                          Mean   : 228709                      Mean   :1977  
                                                          3rd Qu.: 259487                      3rd Qu.:1999  
                                                          Max.   :2220135                      Max.   :2020  
                                                                                                             
  acousticness     danceability        energy          instrumentalness       liveness         loudness      
 Min.   :0.0000   Min.   :0.0000   Min.   :0.0000201   Min.   :0.0000000   Min.   :0.0000   Min.   :-41.808  
 1st Qu.:0.0854   1st Qu.:0.4560   1st Qu.:0.2990000   1st Qu.:0.0000000   1st Qu.:0.0941   1st Qu.:-13.313  
 Median :0.4210   Median :0.5770   Median :0.5140000   Median :0.0000437   Median :0.1290   Median : -9.721  
 Mean   :0.4591   Mean   :0.5653   Mean   :0.5094580   Mean   :0.1095927   Mean   :0.1891   Mean   :-10.491  
 3rd Qu.:0.8300   3rd Qu.:0.6860   3rd Qu.:0.7240000   3rd Qu.:0.0102000   3rd Qu.:0.2380   3rd Qu.: -6.571  
 Max.   :0.9960   Max.   :0.9880   Max.   :0.9990000   Max.   :1.0000000   Max.   :0.9930   Max.   : -0.007  
                                                                                                             
  speechiness         tempo           valence            mode             key           popularity    
 Min.   :0.0000   Min.   :  0.00   Min.   :0.0000   Min.   :0.0000   Min.   : 0.000   Min.   :  1.00  
 1st Qu.:0.0337   1st Qu.: 95.03   1st Qu.:0.3480   1st Qu.:0.0000   1st Qu.: 2.000   1st Qu.: 38.00  
 Median :0.0424   Median :115.67   Median :0.5660   Median :1.0000   Median : 5.000   Median : 57.00  
 Mean   :0.0740   Mean   :117.76   Mean   :0.5519   Mean   :0.7041   Mean   : 5.195   Mean   : 50.38  
 3rd Qu.:0.0650   3rd Qu.:135.09   3rd Qu.:0.7650   3rd Qu.:1.0000   3rd Qu.: 8.000   3rd Qu.: 66.00  
 Max.   :0.9630   Max.   :216.30   Max.   :0.9910   Max.   :1.0000   Max.   :11.000   Max.   :100.00  
                                                                                                      
    explicit           decade    
 Min.   :0.00000   1960s  :2018  
 1st Qu.:0.00000   1970s  :2017  
 Median :0.00000   1980s  :2000  
 Mean   :0.07689   2000s  :1992  
 3rd Qu.:0.00000   1990s  :1986  
 Max.   :1.00000   1950s  :1974  
                   (Other):4959  

ggplot(data2, aes(x = year, y = popularity, color = decade)) +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  xlab("Year") +
  ylab("Popularity") +
  ggtitle("Popularity of Songs by Year and Decade")+
  theme_minimal()

top_popular <- data2 %>%
  group_by(year) %>%
  filter(rank(desc(popularity)) == 1) %>%
  ungroup()
#https://www.r-bloggers.com/2022/07/how-to-rank-by-group-in-r/
ggplot(top_popular, aes(x = year, y = popularity, color = decade)) +
  geom_line() +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  labs(x = "Year", y = "Popularity", title = "Popularity of the Most Popular Songs by Year and Decade")+
  theme_minimal()

top_duration <- data2 %>%
  group_by(year) %>%
  filter(rank(desc(duration_ms)) == 1) %>%
  ungroup()

ggplot(top_duration, aes(x = year, y = duration_ms, color = decade)) +
  geom_line() +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  labs(x = "Year", y = "Duaration (ms)", title = "Duartion of the Most Popular Songs by Year and Decade")+
  theme_minimal()

top_tempo <- data2 %>%
  group_by(year) %>%
  filter(rank(desc(tempo)) == 1) %>%
  ungroup()

ggplot(top_tempo, aes(x = year, y = tempo, color = decade)) +
  geom_line() +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  labs(x = "Year", y = "Tempo", title = "Tempo of the Most Popular Songs by Year and Decade")+
  theme_minimal()

top_speechiness <- data2 %>%
  group_by(year) %>%
  filter(rank(desc(speechiness)) == 1) %>%
  ungroup()

ggplot(top_speechiness, aes(x = year, y = speechiness, color = decade)) +
  geom_line() +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  labs(x = "Year", y = "Speechiness", title = "Speechiness of the Most Popular Songs by Year and Decade")+
  theme_minimal()

ggplot(data2, aes(x = year, y = speechiness, color = decade)) +
  geom_jitter(width = 0.5) +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  xlab("Year") +
  ylab("Speechiness") +
  ggtitle("Speechiness of Songs by Year and Decade")+
  theme_minimal()

ggplot(data2, aes(x = year, y = duration_ms, color = decade)) +
  geom_point() +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  xlab("Year") +
  ylab("Duration (ms)") +
  ggtitle("Duration of Songs by Year and Decade")+
  theme_minimal()

ggplot(data2, aes(x = year, y = tempo, color = decade)) +
  geom_jitter(width = 0.5) +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  xlab("Year") +
  ylab("Tempo (bps)") +
  ggtitle("Tempo of Songs by Year and Decade")+
  theme_minimal()

set.seed(210191)

split <- 0.80

rows <- nrow(data2)

train.entries <- sample(rows, rows * split)

data2.train <- data2[train.entries, ]

data2.valid <- data2[-train.entries, ]


model <- lm(popularity ~ year+ duration_ms + tempo + speechiness, data = data2.train)

summary(model)

Call:
lm(formula = popularity ~ year + duration_ms + tempo + speechiness, 
    data = data2.train)

Residuals:
    Min      1Q  Median      3Q     Max 
-34.259  -6.827  -0.939   6.281  39.213 

Coefficients:
                   Estimate      Std. Error  t value             Pr(>|t|)    
(Intercept) -1448.582646427     6.982888161 -207.447 < 0.0000000000000002 ***
year            0.756762556     0.003559008  212.633 < 0.0000000000000002 ***
duration_ms     0.000006470     0.000001012    6.392       0.000000000169 ***
tempo           0.015136120     0.002991576    5.060       0.000000425680 ***
speechiness    -8.907985506     0.862268090  -10.331 < 0.0000000000000002 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 10.43 on 13551 degrees of freedom
Multiple R-squared:  0.7779,    Adjusted R-squared:  0.7778 
F-statistic: 1.186e+04 on 4 and 13551 DF,  p-value: < 0.00000000000000022

model <- step(model, trace = 0)

summary(model)

Call:
lm(formula = popularity ~ year + duration_ms + tempo + speechiness, 
    data = data2.train)

Residuals:
    Min      1Q  Median      3Q     Max 
-34.259  -6.827  -0.939   6.281  39.213 

Coefficients:
                   Estimate      Std. Error  t value             Pr(>|t|)    
(Intercept) -1448.582646427     6.982888161 -207.447 < 0.0000000000000002 ***
year            0.756762556     0.003559008  212.633 < 0.0000000000000002 ***
duration_ms     0.000006470     0.000001012    6.392       0.000000000169 ***
tempo           0.015136120     0.002991576    5.060       0.000000425680 ***
speechiness    -8.907985506     0.862268090  -10.331 < 0.0000000000000002 ***
---
Signif. codes:  0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1

Residual standard error: 10.43 on 13551 degrees of freedom
Multiple R-squared:  0.7779,    Adjusted R-squared:  0.7778 
F-statistic: 1.186e+04 on 4 and 13551 DF,  p-value: < 0.00000000000000022

data2.train <- data2.train %>%
  mutate(yhat = predict(model, newdata = data2.train)) %>%
  mutate(residual = popularity - yhat)

data2.valid <- data2.valid %>%
  mutate(yhat = predict(model, newdata = data2.valid)) %>%
  mutate(residual = popularity - yhat)


options(scipen = 999)

ggplot(data2.valid, aes(x = yhat, y = residual, color = decade)) +
  geom_jitter(data = filter(data2.valid, yhat <= 2020), width = 2) +
  geom_hline(aes(yintercept = 0), linetype = "dashed", color = "blue") +
  scale_color_manual(values = rainbow(10, start = 0.1, end = 0.9)) +
  xlab("Predicted popularity") +
  ylab("Residuals") +
  ggtitle("Residual Plot")+
  theme_minimal()

NA
NA
NA
NA

sd(data2.valid$residual)

[1] 10.1326

sd(data2.train$residual)

[1] 10.42366