Load the data:
library(readxl)
spotify_songs <- read_excel("spotify/spotify_songs.xlsx")
## New names:
## • `` -> `...7`
head(spotify_songs)
## # A tibble: 6 × 24
## track_id track_name track_artist track_popularity track_album_id
## <chr> <chr> <chr> <dbl> <chr>
## 1 6f807x0ima9a1j3VPbc7VN I Don't C… Ed Sheeran 66 2oCs0DGTsRO98…
## 2 0r7CVbZTWZgbTCYdfa2P31 Memories … Maroon 5 67 63rPSO264uRjW…
## 3 1z1Hg7Vb0AhHDiEmnDE79l All the T… Zara Larsson 70 1HoSmj2eLcsrR…
## 4 75FpbthrwQmzHlBJLuGdC7 Call You … The Chainsm… 60 1nqYsOef1yKKu…
## 5 1e8PAfcKUYoKkxPhrHqw4x Someone Y… Lewis Capal… 69 7m7vv9wlQ4i0L…
## 6 7fvUMiyapMsRRxr07cU8Ef Beautiful… Ed Sheeran 67 2yiy9cd2QktrN…
## # ℹ 19 more variables: track_album_name <chr>, ...7 <lgl>, playlist_name <chr>,
## # playlist_id <chr>, playlist_genre <chr>, playlist_subgenre <chr>,
## # danceability <dbl>, energy <dbl>, key <dbl>, loudness <dbl>, mode <dbl>,
## # speechiness <dbl>, acousticness <dbl>, instrumentalness <dbl>,
## # liveness <dbl>, valence <dbl>, tempo <dbl>, duration_ms <dbl>,
## # track_album_release_date <dbl>
str(spotify_songs)
## tibble [32,833 × 24] (S3: tbl_df/tbl/data.frame)
## $ track_id : chr [1:32833] "6f807x0ima9a1j3VPbc7VN" "0r7CVbZTWZgbTCYdfa2P31" "1z1Hg7Vb0AhHDiEmnDE79l" "75FpbthrwQmzHlBJLuGdC7" ...
## $ track_name : chr [1:32833] "I Don't Care (with Justin Bieber) - Loud Luxury Remix" "Memories - Dillon Francis Remix" "All the Time - Don Diablo Remix" "Call You Mine - Keanu Silva Remix" ...
## $ track_artist : chr [1:32833] "Ed Sheeran" "Maroon 5" "Zara Larsson" "The Chainsmokers" ...
## $ track_popularity : num [1:32833] 66 67 70 60 69 67 62 69 68 67 ...
## $ track_album_id : chr [1:32833] "2oCs0DGTsRO98Gh5ZSl2Cx" "63rPSO264uRjW1X5E6cWv6" "1HoSmj2eLcsrR0vE9gThr4" "1nqYsOef1yKKuGOVchbsk6" ...
## $ track_album_name : chr [1:32833] "I Don't Care (with Justin Bieber) [Loud Luxury Remix]" "Memories (Dillon Francis Remix)" "All the Time (Don Diablo Remix)" "Call You Mine - The Remixes" ...
## $ ...7 : logi [1:32833] NA NA NA NA NA NA ...
## $ playlist_name : chr [1:32833] "Pop Remix" "Pop Remix" "Pop Remix" "Pop Remix" ...
## $ playlist_id : chr [1:32833] "37i9dQZF1DXcZDD7cfEKhW" "37i9dQZF1DXcZDD7cfEKhW" "37i9dQZF1DXcZDD7cfEKhW" "37i9dQZF1DXcZDD7cfEKhW" ...
## $ playlist_genre : chr [1:32833] "pop" "pop" "pop" "pop" ...
## $ playlist_subgenre : chr [1:32833] "dance pop" "dance pop" "dance pop" "dance pop" ...
## $ danceability : num [1:32833] 0.748 0.726 0.675 0.718 0.65 0.675 0.449 0.542 0.594 0.642 ...
## $ energy : num [1:32833] 0.916 0.815 0.931 0.93 0.833 0.919 0.856 0.903 0.935 0.818 ...
## $ key : num [1:32833] 6 11 1 7 1 8 5 4 8 2 ...
## $ loudness : num [1:32833] -2.63 -4.97 -3.43 -3.78 -4.67 ...
## $ mode : num [1:32833] 1 1 0 1 1 1 0 0 1 1 ...
## $ speechiness : num [1:32833] 0.0583 0.0373 0.0742 0.102 0.0359 0.127 0.0623 0.0434 0.0565 0.032 ...
## $ acousticness : num [1:32833] 0.102 0.0724 0.0794 0.0287 0.0803 0.0799 0.187 0.0335 0.0249 0.0567 ...
## $ instrumentalness : num [1:32833] 0.00 4.21e-03 2.33e-05 9.43e-06 0.00 0.00 0.00 4.83e-06 3.97e-06 0.00 ...
## $ liveness : num [1:32833] 0.0653 0.357 0.11 0.204 0.0833 0.143 0.176 0.111 0.637 0.0919 ...
## $ valence : num [1:32833] 0.518 0.693 0.613 0.277 0.725 0.585 0.152 0.367 0.366 0.59 ...
## $ tempo : num [1:32833] 122 100 124 122 124 ...
## $ duration_ms : num [1:32833] 194754 162600 176616 169093 189052 ...
## $ track_album_release_date: num [1:32833] 43630 43812 43651 43665 43529 ...
summary(spotify_songs)
## track_id track_name track_artist track_popularity
## Length:32833 Length:32833 Length:32833 Min. : 0.00
## Class :character Class :character Class :character 1st Qu.: 24.00
## Mode :character Mode :character Mode :character Median : 45.00
## Mean : 42.48
## 3rd Qu.: 62.00
## Max. :100.00
##
## track_album_id track_album_name ...7 playlist_name
## Length:32833 Length:32833 Mode:logical Length:32833
## Class :character Class :character NA's:32833 Class :character
## Mode :character Mode :character Mode :character
##
##
##
##
## playlist_id playlist_genre playlist_subgenre danceability
## Length:32833 Length:32833 Length:32833 Min. :0.0000
## Class :character Class :character Class :character 1st Qu.:0.5630
## Mode :character Mode :character Mode :character Median :0.6720
## Mean :0.6548
## 3rd Qu.:0.7610
## Max. :0.9830
##
## energy key loudness mode
## Min. :0.000175 Min. : 0.000 Min. :-46.448 Min. :0.0000
## 1st Qu.:0.581000 1st Qu.: 2.000 1st Qu.: -8.171 1st Qu.:0.0000
## Median :0.721000 Median : 6.000 Median : -6.166 Median :1.0000
## Mean :0.698619 Mean : 5.374 Mean : -6.720 Mean :0.5657
## 3rd Qu.:0.840000 3rd Qu.: 9.000 3rd Qu.: -4.645 3rd Qu.:1.0000
## Max. :1.000000 Max. :11.000 Max. : 1.275 Max. :1.0000
##
## speechiness acousticness instrumentalness liveness
## Min. :0.0000 Min. :0.0000 Min. :0.0000000 Min. :0.0000
## 1st Qu.:0.0410 1st Qu.:0.0151 1st Qu.:0.0000000 1st Qu.:0.0927
## Median :0.0625 Median :0.0804 Median :0.0000161 Median :0.1270
## Mean :0.1071 Mean :0.1753 Mean :0.0847472 Mean :0.1902
## 3rd Qu.:0.1320 3rd Qu.:0.2550 3rd Qu.:0.0048300 3rd Qu.:0.2480
## Max. :0.9180 Max. :0.9940 Max. :0.9940000 Max. :0.9960
##
## valence tempo duration_ms track_album_release_date
## Min. :0.0000 Min. : 0.00 Min. : 4000 Min. : 1960
## 1st Qu.:0.3310 1st Qu.: 99.96 1st Qu.:187819 1st Qu.:39448
## Median :0.5120 Median :121.98 Median :216000 Median :42608
## Mean :0.5106 Mean :120.88 Mean :225800 Mean :38946
## 3rd Qu.:0.6930 3rd Qu.:133.92 3rd Qu.:253585 3rd Qu.:43574
## Max. :0.9910 Max. :239.44 Max. :517810 Max. :43859
## NA's :31
library(ggplot2)
boxplot(spotify_songs$danceability)
boxplot(spotify_songs$instrumentalness)
boxplot(spotify_songs$speechiness)
ggplot(spotify_songs, aes(x=track_popularity)) +
geom_histogram(binwidth=5, fill="blue", alpha=0.7) +
ggtitle("Dsitribution of Track Popularity") +
xlab("Rating") +
ylab("Frequency")
Hard to tell of any patterns base off of the summary
Check for missing and inconsistent data
To not take up space, this is the code I used: is.na(spotify_songs) No missing data
set.seed(2023)
train_indices <- sample(1:NROW(spotify_songs),NROW(spotify_songs)*0.70)
# Training set
train_data <- spotify_songs[train_indices, ]
# Test set
test_data <- spotify_songs[-train_indices, ]
# Train the linear regression model with multiple variables
lm_model <- lm(track_popularity ~ speechiness + danceability + energy + acousticness + instrumentalness + liveness, data = train_data)
summary(lm_model)
##
## Call:
## lm(formula = track_popularity ~ speechiness + danceability +
## energy + acousticness + instrumentalness + liveness, data = train_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -55.045 -17.904 3.241 19.252 56.726
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 44.2315 1.2023 36.789 < 2e-16 ***
## speechiness -4.9575 1.6336 -3.035 0.00241 **
## danceability 10.3054 1.1465 8.989 < 2e-16 ***
## energy -9.2767 1.0706 -8.665 < 2e-16 ***
## acousticness 5.2763 0.8713 6.056 1.42e-09 ***
## instrumentalness -17.1992 0.7217 -23.833 < 2e-16 ***
## liveness -5.0517 1.0734 -4.706 2.54e-06 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 24.44 on 22976 degrees of freedom
## Multiple R-squared: 0.04052, Adjusted R-squared: 0.04027
## F-statistic: 161.7 on 6 and 22976 DF, p-value: < 2.2e-16
coefficients <- coef(lm_model)
Based on the results, it looks instrumentalness could have the strongest correlation with track_popularity as it is the furthest from 0.
lm_track_popularity_train <- mean((lm_model$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for Linear Model:", round(lm_track_popularity_train, 2)))
## [1] "Training MSE for Linear Model: 597.14"
We get an MSE of 597.14, which means the model is very inaccurate at predicting the track_popularity
library(kknn)
knn_model_5 <- kknn(track_popularity ~ speechiness, train = train_data, test = train_data, k = 5)
summary(knn_model_5)
##
## Call:
## kknn(formula = track_popularity ~ speechiness, train = train_data, test = train_data, k = 5)
##
## Response: "continuous"
knn_model_10 <- kknn(track_popularity ~ speechiness, train = train_data, test = train_data, k = 10)
summary(knn_model_10)
##
## Call:
## kknn(formula = track_popularity ~ speechiness, train = train_data, test = train_data, k = 10)
##
## Response: "continuous"
knn_model_15 <- kknn(track_popularity ~ speechiness, train = train_data, test = train_data, k = 15)
summary(knn_model_15)
##
## Call:
## kknn(formula = track_popularity ~ speechiness, train = train_data, test = train_data, k = 15)
##
## Response: "continuous"
knn_track_popularity_train_5 <- mean((knn_model_5$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 5:", round(knn_track_popularity_train_5, 2)))
## [1] "Training MSE for KNN 5: 694.63"
knn_track_popularity_train_10 <- mean((knn_model_10$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 10:", round(knn_track_popularity_train_10, 2)))
## [1] "Training MSE for KNN 10: 622.64"
knn_track_popularity_train_15 <- mean((knn_model_15$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 15:", round(knn_track_popularity_train_15, 2)))
## [1] "Training MSE for KNN 15: 603.58"
The MSE varied slightly between different k values. K = 15 got us the lowest MSE at 603.58
KNN MSE
print(paste("Training MSE for KNN 10:", round(knn_track_popularity_train_10, 2)))
## [1] "Training MSE for KNN 10: 622.64"
LM MSE
print(paste("Training MSE for Linear Model:", round(lm_track_popularity_train, 2)))
## [1] "Training MSE for Linear Model: 597.14"
The KNN model had a better MSE score. The MSE is still not great in terms of correlation, but KNN was still better
The linear model is easier to interpret and is an easier fit for data. But knn has a more flexible approach. The disadvantages are that linear regression will be a poor fit if the data does not form a correlation. KNN is a bit more complex to interpret.
I would recommend KNN since there doesn’t seem to be a correlation between variables.
The MSE seemed to go down the higher the k value went (though not by much).
Linear assumptions are: relationship between x and y is linear, variance of residual is the same for any value of x, observations are independent of each other, and for any fixed value of X, Y is normally distributed.
We can tell if the residuals are normally dsitributed through this plot
plot(lm_model, 2)
The plot should ideally have the residuals following in a straight line. In this case, they are not.
lm_model_interation <- lm(track_popularity ~ speechiness * danceability, data = train_data)
summary(lm_model_interation)
##
## Call:
## lm(formula = track_popularity ~ speechiness * danceability, data = train_data)
##
## Residuals:
## Min 1Q Median 3Q Max
## -47.614 -18.329 3.049 19.576 57.080
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 37.072 1.106 33.505 < 2e-16 ***
## speechiness -19.821 7.866 -2.520 0.0117 *
## danceability 8.336 1.693 4.924 8.52e-07 ***
## speechiness:danceability 28.344 11.415 2.483 0.0130 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 24.89 on 22979 degrees of freedom
## Multiple R-squared: 0.004637, Adjusted R-squared: 0.004507
## F-statistic: 35.69 on 3 and 22979 DF, p-value: < 2.2e-16
There is less variance explained in the interaction model than the no interaction. I would say this model is less reliable and a worse fit for my data.
boxplot(spotify_songs$danceability,data=spotify_songs, main="Outliers Check",
xlab="Dancebaility", ylab="Frequency")
We can see that there are multiple outliers in the lower values
lm_model_win_before <- lm(track_popularity ~ danceability, data = spotify_songs)
summary(lm_model_win_before)
##
## Call:
## lm(formula = track_popularity ~ danceability, data = spotify_songs)
##
## Residuals:
## Min 1Q Median 3Q Max
## -46.024 -18.415 2.934 19.480 57.105
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 35.1757 0.6361 55.30 <2e-16 ***
## danceability 11.1497 0.9484 11.76 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 24.93 on 32831 degrees of freedom
## Multiple R-squared: 0.004192, Adjusted R-squared: 0.004162
## F-statistic: 138.2 on 1 and 32831 DF, p-value: < 2.2e-16
lower_bound_zn <- quantile(spotify_songs$danceability, 0.01, na.rm = TRUE)
upper_bound_zn <- quantile(spotify_songs$danceability, 0.99, na.rm = TRUE)
spotify_songs$danceability[spotify_songs$danceability < lower_bound_zn] <- lower_bound_zn
spotify_songs$danceability[spotify_songs$danceability > upper_bound_zn] <- upper_bound_zn
boxplot(spotify_songs$danceability,data=spotify_songs, main="Outliers Check",
xlab="Dancebaility", ylab="Frequency")
The boxplot indicates there is no outliers after winsorizaing the data.
lm_model_win <- lm(track_popularity ~ danceability, data = spotify_songs)
summary(lm_model_win)
##
## Call:
## lm(formula = track_popularity ~ danceability, data = spotify_songs)
##
## Residuals:
## Min 1Q Median 3Q Max
## -45.549 -18.410 2.944 19.475 57.135
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 35.0478 0.6438 54.44 <2e-16 ***
## danceability 11.3403 0.9600 11.81 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 24.93 on 32831 degrees of freedom
## Multiple R-squared: 0.004233, Adjusted R-squared: 0.004202
## F-statistic: 139.6 on 1 and 32831 DF, p-value: < 2.2e-16
There is slightly more variance explained as indicated by R^2
Feature scaling helps prevent dominant numbers from having a greater influence in the KNN model.
I would recommend KNN models for this dataset, since linear regression doesn’t seem to be a true fit. The assumptions are not met.