The problem statement I will be addressing is what rating(s) categories have the biggest influence on track popularity (if any). It will help solve out if there is a althgromitic system to creating the most popular song.
I plan to address this problem statement by running multiple linear and KNN analysis on the spotify data sheet with each variable and track popularity to see which have the highest correlation.
By finding a variable with the highest correlation and finding if the relationship is significant, we can address the problem of finding correlations with track popularity.
My analysis can help future upcoming artists write a song if they’re goal is acheiving song popularity.
KKNN Package, GGplot2, and ReadXL
# Installing libraries
# KKNN
library(kknn)
# GG Plot
library(ggplot2)
# Read Excel
library(readxl)
# KnitR
library(knitr)
# SVM
library("e1071")
# Tree Models
library(rpart)
This data set was obtained by Charlie Thompson, Josiah Parry, Donal Phipps, and Tom Wolff for use on R
The purpose of this data package was to make it easier for the average consumer to get your own data or metadata around songs from Spotify’s API. This data set contains 23 variables, and there is some missing values on variables such as the release date which is strange.
Load the data:
spotify_songs <- read_excel("spotify/spotify_songs.xlsx")
## New names:
## • `` -> `...7`
# Check for missing values
colSums(is.na(spotify_songs))
## track_id track_name track_artist
## 0 1 0
## track_popularity track_album_id track_album_name
## 0 0 1
## ...7 playlist_name playlist_id
## 32833 0 0
## playlist_genre playlist_subgenre danceability
## 0 0 0
## energy key loudness
## 0 0 0
## mode speechiness acousticness
## 0 0 0
## instrumentalness liveness valence
## 0 0 0
## tempo duration_ms track_album_release_date
## 0 0 31
# Track_popularity does not have outliers
# Danceability
lower_bound_danceability <- quantile(spotify_songs$danceability, 0.01, na.rm = TRUE)
upper_bound_danceability <- quantile(spotify_songs$danceability, 0.99, na.rm = TRUE)
spotify_songs$danceability[spotify_songs$danceability < lower_bound_danceability] <- lower_bound_danceability
spotify_songs$danceability[spotify_songs$danceability > upper_bound_danceability] <- upper_bound_danceability
# Energy
lower_bound_energy <- quantile(spotify_songs$energy, 0.10, na.rm = TRUE)
upper_bound_energy <- quantile(spotify_songs$energy, 0.90, na.rm = TRUE)
spotify_songs$energy[spotify_songs$energy < lower_bound_energy] <- lower_bound_energy
spotify_songs$energy[spotify_songs$energy > upper_bound_energy] <- upper_bound_energy
# Loudness
lower_bound_loudness <- quantile(spotify_songs$loudness, 0.05, na.rm = TRUE)
upper_bound_loudness <- quantile(spotify_songs$loudness, 0.99, na.rm = TRUE)
spotify_songs$loudness[spotify_songs$loudness < lower_bound_loudness] <- lower_bound_loudness
spotify_songs$loudness[spotify_songs$loudness > upper_bound_loudness] <- upper_bound_loudness
# Mode does not have outliers
# Speechiness
lower_bound_speechiness <- quantile(spotify_songs$speechiness, 0.01, na.rm = TRUE)
upper_bound_speechiness <- quantile(spotify_songs$speechiness, 0.90, na.rm = TRUE)
spotify_songs$speechiness[spotify_songs$speechiness < lower_bound_speechiness] <- lower_bound_speechiness
spotify_songs$speechiness[spotify_songs$speechiness > upper_bound_speechiness] <- upper_bound_speechiness
# Acousticness
lower_bound_acousticness <- quantile(spotify_songs$acousticness, 0.01, na.rm = TRUE)
upper_bound_acousticness <- quantile(spotify_songs$acousticness, 0.90, na.rm = TRUE)
spotify_songs$acousticness[spotify_songs$acousticness < lower_bound_acousticness] <- lower_bound_acousticness
spotify_songs$acousticness[spotify_songs$acousticness > upper_bound_acousticness] <- upper_bound_acousticness
# Instrumentalness
lower_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.01, na.rm = TRUE)
upper_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.75, na.rm = TRUE)
spotify_songs$instrumentalness[spotify_songs$instrumentalness < lower_bound_instrumentalness] <- lower_bound_instrumentalness
spotify_songs$instrumentalness[spotify_songs$instrumentalness > upper_bound_instrumentalness] <- upper_bound_instrumentalness
# Liveness
lower_bound_liveness <- quantile(spotify_songs$liveness, 0.01, na.rm = TRUE)
upper_bound_liveness <- quantile(spotify_songs$liveness, 0.90, na.rm = TRUE)
spotify_songs$liveness[spotify_songs$liveness < lower_bound_liveness] <- lower_bound_liveness
spotify_songs$liveness[spotify_songs$liveness > upper_bound_liveness] <- upper_bound_liveness
# Valence does not have outliers
# Tempo
lower_bound_tempo <- quantile(spotify_songs$tempo, 0.05, na.rm = TRUE)
upper_bound_tempo <- quantile(spotify_songs$tempo, 0.90, na.rm = TRUE)
spotify_songs$tempo[spotify_songs$tempo < lower_bound_tempo] <- lower_bound_tempo
spotify_songs$tempo[spotify_songs$tempo > upper_bound_tempo] <- upper_bound_tempo
# Duration_ms
lower_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.01, na.rm = TRUE)
upper_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.95, na.rm = TRUE)
spotify_songs$duration_ms[spotify_songs$duration_ms < lower_bound_duration_ms] <- lower_bound_duration_ms
spotify_songs$duration_ms[spotify_songs$duration_ms > upper_bound_duration_ms] <- upper_bound_duration_ms
Visualizations
# Outlier check
boxplot(spotify_songs$danceability, spotify_songs$energy, spotify_songs$key, spotify_songs$loudness, spotify_songs$mode, spotify_songs$speechiness, spotify_songs$acousticness, spotify_songs$instrumentalness, spotify_songs$liveness, spotify_songs$valence, spotify_songs$tempo, spotify_songs$track_popularity)
# Loudness or valence seems to have the closet relation with trackpopularity
ggplot(spotify_songs, aes(x=loudness, y=track_popularity)) +
geom_point(size=2, shape=23) +
geom_smooth(method=lm)
## `geom_smooth()` using formula = 'y ~ x'
ggplot(spotify_songs, aes(x=valence, y=track_popularity)) +
geom_point(size=2, shape=23) +
geom_smooth(method=lm)
## `geom_smooth()` using formula = 'y ~ x'
ggplot(spotify_songs, aes(x=track_popularity)) +
geom_histogram(binwidth=5, fill="blue", alpha=0.7) +
ggtitle("Distribution of Track Popularity") +
xlab("Rating") +
ylab("Frequency")
knitr::kable(head(spotify_songs[, 1:5]), "simple")
| track_id | track_name | track_artist | track_popularity | track_album_id |
|---|---|---|---|---|
| 6f807x0ima9a1j3VPbc7VN | I Don’t Care (with Justin Bieber) - Loud Luxury Remix | Ed Sheeran | 66 | 2oCs0DGTsRO98Gh5ZSl2Cx |
| 0r7CVbZTWZgbTCYdfa2P31 | Memories - Dillon Francis Remix | Maroon 5 | 67 | 63rPSO264uRjW1X5E6cWv6 |
| 1z1Hg7Vb0AhHDiEmnDE79l | All the Time - Don Diablo Remix | Zara Larsson | 70 | 1HoSmj2eLcsrR0vE9gThr4 |
| 75FpbthrwQmzHlBJLuGdC7 | Call You Mine - Keanu Silva Remix | The Chainsmokers | 60 | 1nqYsOef1yKKuGOVchbsk6 |
| 1e8PAfcKUYoKkxPhrHqw4x | Someone You Loved - Future Humans Remix | Lewis Capaldi | 69 | 7m7vv9wlQ4i0LFuJiE2zsQ |
| 7fvUMiyapMsRRxr07cU8Ef | Beautiful People (feat. Khalid) - Jack Wins Remix | Ed Sheeran | 67 | 2yiy9cd2QktrNvWC2EUi0k |
knitr::kable(head(spotify_songs[, 6:10]), "simple")
| track_album_name | …7 | playlist_name | playlist_id | playlist_genre |
|---|---|---|---|---|
| I Don’t Care (with Justin Bieber) [Loud Luxury Remix] | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
| Memories (Dillon Francis Remix) | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
| All the Time (Don Diablo Remix) | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
| Call You Mine - The Remixes | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
| Someone You Loved (Future Humans Remix) | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
| Beautiful People (feat. Khalid) [Jack Wins Remix] | NA | Pop Remix | 37i9dQZF1DXcZDD7cfEKhW | pop |
knitr::kable(head(spotify_songs[, 11:16]), "simple")
| playlist_subgenre | danceability | energy | key | loudness | mode |
|---|---|---|---|---|---|
| dance pop | 0.748 | 0.916 | 6 | -2.634 | 1 |
| dance pop | 0.726 | 0.815 | 11 | -4.969 | 1 |
| dance pop | 0.675 | 0.918 | 1 | -3.432 | 0 |
| dance pop | 0.718 | 0.918 | 7 | -3.778 | 1 |
| dance pop | 0.650 | 0.833 | 1 | -4.672 | 1 |
| dance pop | 0.675 | 0.918 | 8 | -5.385 | 1 |
knitr::kable(head(spotify_songs[, 17:22]), "simple")
| speechiness | acousticness | instrumentalness | liveness | valence | tempo |
|---|---|---|---|---|---|
| 0.0583 | 0.1020 | 0.00e+00 | 0.0653 | 0.518 | 122.036 |
| 0.0373 | 0.0724 | 4.21e-03 | 0.3570 | 0.693 | 99.972 |
| 0.0742 | 0.0794 | 2.33e-05 | 0.1100 | 0.613 | 124.008 |
| 0.1020 | 0.0287 | 9.40e-06 | 0.2040 | 0.277 | 121.956 |
| 0.0359 | 0.0803 | 0.00e+00 | 0.0833 | 0.725 | 123.976 |
| 0.1270 | 0.0799 | 0.00e+00 | 0.1430 | 0.585 | 124.982 |
The variables I thought were going to be a correlation turned out to be least correlated with track_popularity according to the scatter plots. There were a lot of outliers, so it makes me wonder if there is a pattern to creating the most popularity track at all. I can still test out different models as I planned, but I’ll primarily focus on loudness and valence.
The scatter plot will probably be the most helpful to answer my question. I can plot the variables and see if there is a correlation just by initial looks.
I’m not sure what model will be the best for my data yet. I think linear regression will be the best, but even with the most correlated variable, the correlation was lacking at best. I worry it might be forcing a fit too much.
I will spilt my data 70/30 train/test retrospectively.
set.seed(2023)
train_indices <- sample(1:NROW(spotify_songs),NROW(spotify_songs)*0.70)
# Create the training set
train_data <- spotify_songs[train_indices, ]
# Create the testing set
test_data <- spotify_songs[-train_indices, ]
# Creating the model
lm_model <- lm(track_popularity ~ loudness + valence, data = spotify_songs)
# Summary stats
summary(lm_model)
##
## Call:
## lm(formula = track_popularity ~ loudness + valence, data = spotify_songs)
##
## Residuals:
## Min 1Q Median 3Q Max
## -46.497 -18.386 3.028 19.490 58.765
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 44.61767 0.48794 91.440 <2e-16 ***
## loudness 0.57656 0.05219 11.047 <2e-16 ***
## valence 3.27328 0.59058 5.543 3e-08 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 24.92 on 32830 degrees of freedom
## Multiple R-squared: 0.004803, Adjusted R-squared: 0.004743
## F-statistic: 79.23 on 2 and 32830 DF, p-value: < 2.2e-16
# Coefficient stats
coefficients <- coef(lm_model)
Testing the MSE
lm_track_popularity <- mean((lm_model$fitted.values - spotify_songs$track_popularity)^2)
print(paste("Training MSE for Linear Model:", round(lm_track_popularity, 2)))
## [1] "Training MSE for Linear Model: 621.19"
# KNN modeling wtih k values 5, 10, and 15
knn_model_5 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 5)
summary(knn_model_5)
##
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 5)
##
## Response: "continuous"
knn_model_10 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 10)
summary(knn_model_10)
##
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 10)
##
## Response: "continuous"
knn_model_15 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 15)
summary(knn_model_15)
##
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 15)
##
## Response: "continuous"
Testing the MSE
knn_track_popularity_train_5 <- mean((knn_model_5$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 5:", round(knn_track_popularity_train_5, 2)))
## [1] "Training MSE for KNN 5: 282.96"
knn_track_popularity_train_10 <- mean((knn_model_10$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 10:", round(knn_track_popularity_train_10, 2)))
## [1] "Training MSE for KNN 10: 412.26"
knn_track_popularity_train_15 <- mean((knn_model_15$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 15:", round(knn_track_popularity_train_15, 2)))
## [1] "Training MSE for KNN 15: 468.3"
# Creating SVm model
svmfit = svm(track_popularity ~ loudness + valence, data = train_data, kernal = "linear", cost = 10, scale = FALSE)
# Testing with test data to make predictions
predictions <- predict(svmfit, newdata = test_data)
# Finding the accuracy of the svm model
accuracy <- sum(spotify_songs$track_popularity == predictions) / length(spotify_songs$track_popularity)
print(paste("Accuracy:", accuracy))
## [1] "Accuracy: 0"
treefit <- rpart(track_popularity ~ loudness + valence, method = "anova", data=train_data)
summary(treefit)
## Call:
## rpart(formula = track_popularity ~ loudness + valence, data = train_data,
## method = "anova")
## n= 22983
##
## CP nsplit rel error xerror xstd
## 1 0.004983508 0 1 0 0
##
## Node number 1: 22983 observations
## mean=42.47827, MSE=622.3641