Spotify Data

Packages Required

KKNN Package, GGplot2, and ReadXL

# Installing libraries
# KKNN
library(kknn)
# GG Plot
library(ggplot2)
# Read Excel
library(readxl)
# KnitR
library(knitr)
# SVM
library("e1071")
# Tree Models
library(rpart)

Data Preparation

  • This data set was obtained by Charlie Thompson, Josiah Parry, Donal Phipps, and Tom Wolff for use on R

  • The purpose of this data package was to make it easier for the average consumer to get your own data or metadata around songs from Spotify’s API. This data set contains 23 variables, and there is some missing values on variables such as the release date which is strange.

Load the data:

spotify_songs <- read_excel("spotify/spotify_songs.xlsx")
## New names:
## • `` -> `...7`
  • For Data cleaning, I have checked all variables and cleaned up any outliers, missing values, and displayed summaries so that I can understand my data. All the code for this process can be found in the tab “Data Cleaning Work”.

Data Cleaning Work

# Check for missing values

colSums(is.na(spotify_songs))
##                 track_id               track_name             track_artist 
##                        0                        1                        0 
##         track_popularity           track_album_id         track_album_name 
##                        0                        0                        1 
##                     ...7            playlist_name              playlist_id 
##                    32833                        0                        0 
##           playlist_genre        playlist_subgenre             danceability 
##                        0                        0                        0 
##                   energy                      key                 loudness 
##                        0                        0                        0 
##                     mode              speechiness             acousticness 
##                        0                        0                        0 
##         instrumentalness                 liveness                  valence 
##                        0                        0                        0 
##                    tempo              duration_ms track_album_release_date 
##                        0                        0                       31
Looks like we do have some missing values. Thankfully, to answer my questions we will not be needing to use these variables. Next I will check for outliers, and replace them with the average value for that variable. This way my more extreme values will not have a big impact on my models.
# Track_popularity does not have outliers

# Danceability 
lower_bound_danceability <- quantile(spotify_songs$danceability, 0.01, na.rm = TRUE)
upper_bound_danceability <- quantile(spotify_songs$danceability, 0.99, na.rm = TRUE)

spotify_songs$danceability[spotify_songs$danceability < lower_bound_danceability] <- lower_bound_danceability
spotify_songs$danceability[spotify_songs$danceability > upper_bound_danceability] <- upper_bound_danceability

# Energy 
lower_bound_energy <- quantile(spotify_songs$energy, 0.10, na.rm = TRUE)
upper_bound_energy <- quantile(spotify_songs$energy, 0.90, na.rm = TRUE)

spotify_songs$energy[spotify_songs$energy < lower_bound_energy] <- lower_bound_energy
spotify_songs$energy[spotify_songs$energy > upper_bound_energy] <- upper_bound_energy


# Loudness 
lower_bound_loudness <- quantile(spotify_songs$loudness, 0.05, na.rm = TRUE)
upper_bound_loudness <- quantile(spotify_songs$loudness, 0.99, na.rm = TRUE)

spotify_songs$loudness[spotify_songs$loudness < lower_bound_loudness] <- lower_bound_loudness
spotify_songs$loudness[spotify_songs$loudness > upper_bound_loudness] <- upper_bound_loudness

# Mode does not have outliers

# Speechiness 
lower_bound_speechiness <- quantile(spotify_songs$speechiness, 0.01, na.rm = TRUE)
upper_bound_speechiness <- quantile(spotify_songs$speechiness, 0.90, na.rm = TRUE)

spotify_songs$speechiness[spotify_songs$speechiness < lower_bound_speechiness] <- lower_bound_speechiness
spotify_songs$speechiness[spotify_songs$speechiness > upper_bound_speechiness] <- upper_bound_speechiness

# Acousticness
lower_bound_acousticness <- quantile(spotify_songs$acousticness, 0.01, na.rm = TRUE)
upper_bound_acousticness <- quantile(spotify_songs$acousticness, 0.90, na.rm = TRUE)

spotify_songs$acousticness[spotify_songs$acousticness < lower_bound_acousticness] <- lower_bound_acousticness
spotify_songs$acousticness[spotify_songs$acousticness > upper_bound_acousticness] <- upper_bound_acousticness

# Instrumentalness
lower_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.01, na.rm = TRUE)
upper_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.75, na.rm = TRUE)

spotify_songs$instrumentalness[spotify_songs$instrumentalness < lower_bound_instrumentalness] <- lower_bound_instrumentalness
spotify_songs$instrumentalness[spotify_songs$instrumentalness > upper_bound_instrumentalness] <- upper_bound_instrumentalness

# Liveness
lower_bound_liveness <- quantile(spotify_songs$liveness, 0.01, na.rm = TRUE)
upper_bound_liveness <- quantile(spotify_songs$liveness, 0.90, na.rm = TRUE)

spotify_songs$liveness[spotify_songs$liveness < lower_bound_liveness] <- lower_bound_liveness
spotify_songs$liveness[spotify_songs$liveness > upper_bound_liveness] <- upper_bound_liveness

# Valence does not have outliers

# Tempo
lower_bound_tempo <- quantile(spotify_songs$tempo, 0.05, na.rm = TRUE)
upper_bound_tempo <- quantile(spotify_songs$tempo, 0.90, na.rm = TRUE)

spotify_songs$tempo[spotify_songs$tempo < lower_bound_tempo] <- lower_bound_tempo
spotify_songs$tempo[spotify_songs$tempo > upper_bound_tempo] <- upper_bound_tempo

# Duration_ms
lower_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.01, na.rm = TRUE)
upper_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.95, na.rm = TRUE)

spotify_songs$duration_ms[spotify_songs$duration_ms < lower_bound_duration_ms] <- lower_bound_duration_ms
spotify_songs$duration_ms[spotify_songs$duration_ms > upper_bound_duration_ms] <- upper_bound_duration_ms

Visualizations

# Outlier check
boxplot(spotify_songs$danceability, spotify_songs$energy, spotify_songs$key, spotify_songs$loudness, spotify_songs$mode, spotify_songs$speechiness, spotify_songs$acousticness, spotify_songs$instrumentalness, spotify_songs$liveness, spotify_songs$valence, spotify_songs$tempo, spotify_songs$track_popularity)

# Loudness or valence seems to have the closet relation with trackpopularity
ggplot(spotify_songs, aes(x=loudness, y=track_popularity)) +
  geom_point(size=2, shape=23) + 
  geom_smooth(method=lm)
## `geom_smooth()` using formula = 'y ~ x'

ggplot(spotify_songs, aes(x=valence, y=track_popularity)) +
    geom_point(size=2, shape=23) + 
    geom_smooth(method=lm)
## `geom_smooth()` using formula = 'y ~ x'

ggplot(spotify_songs, aes(x=track_popularity)) +
  geom_histogram(binwidth=5, fill="blue", alpha=0.7) +
  ggtitle("Distribution of Track Popularity") +
  xlab("Rating") +
  ylab("Frequency")

Final Look at Data at Cleaning
knitr::kable(head(spotify_songs[, 1:5]), "simple")
track_id track_name track_artist track_popularity track_album_id
6f807x0ima9a1j3VPbc7VN I Don’t Care (with Justin Bieber) - Loud Luxury Remix Ed Sheeran 66 2oCs0DGTsRO98Gh5ZSl2Cx
0r7CVbZTWZgbTCYdfa2P31 Memories - Dillon Francis Remix Maroon 5 67 63rPSO264uRjW1X5E6cWv6
1z1Hg7Vb0AhHDiEmnDE79l All the Time - Don Diablo Remix Zara Larsson 70 1HoSmj2eLcsrR0vE9gThr4
75FpbthrwQmzHlBJLuGdC7 Call You Mine - Keanu Silva Remix The Chainsmokers 60 1nqYsOef1yKKuGOVchbsk6
1e8PAfcKUYoKkxPhrHqw4x Someone You Loved - Future Humans Remix Lewis Capaldi 69 7m7vv9wlQ4i0LFuJiE2zsQ
7fvUMiyapMsRRxr07cU8Ef Beautiful People (feat. Khalid) - Jack Wins Remix Ed Sheeran 67 2yiy9cd2QktrNvWC2EUi0k
knitr::kable(head(spotify_songs[, 6:10]), "simple")
track_album_name …7 playlist_name playlist_id playlist_genre
I Don’t Care (with Justin Bieber) [Loud Luxury Remix] NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
Memories (Dillon Francis Remix) NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
All the Time (Don Diablo Remix) NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
Call You Mine - The Remixes NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
Someone You Loved (Future Humans Remix) NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
Beautiful People (feat. Khalid) [Jack Wins Remix] NA Pop Remix 37i9dQZF1DXcZDD7cfEKhW pop
knitr::kable(head(spotify_songs[, 11:16]), "simple")
playlist_subgenre danceability energy key loudness mode
dance pop 0.748 0.916 6 -2.634 1
dance pop 0.726 0.815 11 -4.969 1
dance pop 0.675 0.918 1 -3.432 0
dance pop 0.718 0.918 7 -3.778 1
dance pop 0.650 0.833 1 -4.672 1
dance pop 0.675 0.918 8 -5.385 1
knitr::kable(head(spotify_songs[, 17:22]), "simple")
speechiness acousticness instrumentalness liveness valence tempo
0.0583 0.1020 0.00e+00 0.0653 0.518 122.036
0.0373 0.0724 4.21e-03 0.3570 0.693 99.972
0.0742 0.0794 2.33e-05 0.1100 0.613 124.008
0.1020 0.0287 9.40e-06 0.2040 0.277 121.956
0.0359 0.0803 0.00e+00 0.0833 0.725 123.976
0.1270 0.0799 0.00e+00 0.1430 0.585 124.982
Summary Information
  • Our character variables include: playlist_name, playlist_id, playlist_genre, playlist_subgenre, track_id, track_name. track_artist, track_album_id, and track_album_name.
  • Our numeric variables range in the range of the variable. Firstly, the numeric variables include: track_popularity, danceability, energy, key, loudness, mode, speechiness, acousticness, instrumentalness, liveness, valence, tempo, and duration_ms. There are varaibles such as energy and acousticness whose median falls between 0-1, variables such as loudness whose median is -6, and other variables such as tempo and duration whose median is in the hundreds and thousands.

Exploreatory Data Analysis

  • The variables I thought were going to be a correlation turned out to be least correlated with track_popularity according to the scatter plots. There were a lot of outliers, so it makes me wonder if there is a pattern to creating the most popularity track at all. I can still test out different models as I planned, but I’ll primarily focus on loudness and valence.

  • The scatter plot will probably be the most helpful to answer my question. I can plot the variables and see if there is a correlation just by initial looks.

  • I’m not sure what model will be the best for my data yet. I think linear regression will be the best, but even with the most correlated variable, the correlation was lacking at best. I worry it might be forcing a fit too much.

  • I will spilt my data 70/30 train/test retrospectively.

set.seed(2023)
train_indices <- sample(1:NROW(spotify_songs),NROW(spotify_songs)*0.70)

# Create the training set
train_data <- spotify_songs[train_indices, ]

# Create the testing set
test_data <- spotify_songs[-train_indices, ]

Modelling

Linear Regression
# Creating the model
lm_model <- lm(track_popularity ~ loudness + valence, data = spotify_songs)
# Summary stats
summary(lm_model)
## 
## Call:
## lm(formula = track_popularity ~ loudness + valence, data = spotify_songs)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -46.497 -18.386   3.028  19.490  58.765 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 44.61767    0.48794  91.440   <2e-16 ***
## loudness     0.57656    0.05219  11.047   <2e-16 ***
## valence      3.27328    0.59058   5.543    3e-08 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 24.92 on 32830 degrees of freedom
## Multiple R-squared:  0.004803,   Adjusted R-squared:  0.004743 
## F-statistic: 79.23 on 2 and 32830 DF,  p-value: < 2.2e-16
# Coefficient stats
coefficients <- coef(lm_model)

Testing the MSE

lm_track_popularity <- mean((lm_model$fitted.values - spotify_songs$track_popularity)^2)
print(paste("Training MSE for Linear Model:", round(lm_track_popularity, 2)))
## [1] "Training MSE for Linear Model: 621.19"
KNN
# KNN modeling wtih k values 5, 10, and 15
knn_model_5 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 5)
summary(knn_model_5)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 5)
## 
## Response: "continuous"
knn_model_10 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 10)
summary(knn_model_10)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 10)
## 
## Response: "continuous"
knn_model_15 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 15)
summary(knn_model_15)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 15)
## 
## Response: "continuous"

Testing the MSE

knn_track_popularity_train_5 <- mean((knn_model_5$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 5:", round(knn_track_popularity_train_5, 2)))
## [1] "Training MSE for KNN 5: 282.96"
knn_track_popularity_train_10 <- mean((knn_model_10$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 10:", round(knn_track_popularity_train_10, 2)))
## [1] "Training MSE for KNN 10: 412.26"
knn_track_popularity_train_15 <- mean((knn_model_15$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 15:", round(knn_track_popularity_train_15, 2)))
## [1] "Training MSE for KNN 15: 468.3"
SVM
# Creating SVm model
svmfit = svm(track_popularity ~ loudness + valence, data = train_data, kernal = "linear", cost = 10, scale = FALSE)
# Testing with test data to make predictions
predictions <- predict(svmfit, newdata = test_data)
# Finding the accuracy of the svm model
accuracy <- sum(spotify_songs$track_popularity == predictions) / length(spotify_songs$track_popularity)
print(paste("Accuracy:", accuracy))
## [1] "Accuracy: 0"
Tree models
treefit <- rpart(track_popularity ~ loudness + valence, method = "anova", data=train_data)
summary(treefit)
## Call:
## rpart(formula = track_popularity ~ loudness + valence, data = train_data, 
##     method = "anova")
##   n= 22983 
## 
##            CP nsplit rel error xerror xstd
## 1 0.004983508      0         1      0    0
## 
## Node number 1: 22983 observations
##   mean=42.47827, MSE=622.3641
  • I only used track popularity, loudness, and valence sin they seemed to have the greatest correlation to predicting track popularity.
  • I think the linear regression model fits best. SVM and Tree models fit very poorly. Additionally, linear regression is the model closet to answering the problem statement.
  • The model that fits best in practice is probably a model I have yet to learn in this class. The data is way to scattered to fit perfectly in any of the models I tried using. I would know what fits best by looking for a low MSE.