Homework 4

Data Exploration

# Installing libraries
# KKNN
library(kknn)
# GG Plot
library(ggplot2)
# Read Excel
library(readxl)
# CorrPlot
library(corrplot)
## corrplot 0.92 loaded
spotify_songs <- read_excel("spotify/spotify_songs.xlsx")
## New names:
## • `` -> `...7`
summary(spotify_songs)
##    track_id          track_name        track_artist       track_popularity
##  Length:32833       Length:32833       Length:32833       Min.   :  0.00  
##  Class :character   Class :character   Class :character   1st Qu.: 24.00  
##  Mode  :character   Mode  :character   Mode  :character   Median : 45.00  
##                                                           Mean   : 42.48  
##                                                           3rd Qu.: 62.00  
##                                                           Max.   :100.00  
##                                                                           
##  track_album_id     track_album_name     ...7         playlist_name     
##  Length:32833       Length:32833       Mode:logical   Length:32833      
##  Class :character   Class :character   NA's:32833     Class :character  
##  Mode  :character   Mode  :character                  Mode  :character  
##                                                                         
##                                                                         
##                                                                         
##                                                                         
##  playlist_id        playlist_genre     playlist_subgenre   danceability   
##  Length:32833       Length:32833       Length:32833       Min.   :0.0000  
##  Class :character   Class :character   Class :character   1st Qu.:0.5630  
##  Mode  :character   Mode  :character   Mode  :character   Median :0.6720  
##                                                           Mean   :0.6548  
##                                                           3rd Qu.:0.7610  
##                                                           Max.   :0.9830  
##                                                                           
##      energy              key            loudness            mode       
##  Min.   :0.000175   Min.   : 0.000   Min.   :-46.448   Min.   :0.0000  
##  1st Qu.:0.581000   1st Qu.: 2.000   1st Qu.: -8.171   1st Qu.:0.0000  
##  Median :0.721000   Median : 6.000   Median : -6.166   Median :1.0000  
##  Mean   :0.698619   Mean   : 5.374   Mean   : -6.720   Mean   :0.5657  
##  3rd Qu.:0.840000   3rd Qu.: 9.000   3rd Qu.: -4.645   3rd Qu.:1.0000  
##  Max.   :1.000000   Max.   :11.000   Max.   :  1.275   Max.   :1.0000  
##                                                                        
##   speechiness      acousticness    instrumentalness       liveness     
##  Min.   :0.0000   Min.   :0.0000   Min.   :0.0000000   Min.   :0.0000  
##  1st Qu.:0.0410   1st Qu.:0.0151   1st Qu.:0.0000000   1st Qu.:0.0927  
##  Median :0.0625   Median :0.0804   Median :0.0000161   Median :0.1270  
##  Mean   :0.1071   Mean   :0.1753   Mean   :0.0847472   Mean   :0.1902  
##  3rd Qu.:0.1320   3rd Qu.:0.2550   3rd Qu.:0.0048300   3rd Qu.:0.2480  
##  Max.   :0.9180   Max.   :0.9940   Max.   :0.9940000   Max.   :0.9960  
##                                                                        
##     valence           tempo         duration_ms     track_album_release_date
##  Min.   :0.0000   Min.   :  0.00   Min.   :  4000   Min.   : 1960           
##  1st Qu.:0.3310   1st Qu.: 99.96   1st Qu.:187819   1st Qu.:39448           
##  Median :0.5120   Median :121.98   Median :216000   Median :42608           
##  Mean   :0.5106   Mean   :120.88   Mean   :225800   Mean   :38946           
##  3rd Qu.:0.6930   3rd Qu.:133.92   3rd Qu.:253585   3rd Qu.:43574           
##  Max.   :0.9910   Max.   :239.44   Max.   :517810   Max.   :43859           
##                                                     NA's   :31
ggplot(spotify_songs, aes(x=track_popularity)) +
  geom_histogram(binwidth=5, fill="blue", alpha=0.7) +
  ggtitle("Dsitribution of Track Popularity") +
  xlab("Rating") +
  ylab("Frequency")

Data Preprocessing

# Check for missing values

colSums(is.na(spotify_songs))
##                 track_id               track_name             track_artist 
##                        0                        1                        0 
##         track_popularity           track_album_id         track_album_name 
##                        0                        0                        1 
##                     ...7            playlist_name              playlist_id 
##                    32833                        0                        0 
##           playlist_genre        playlist_subgenre             danceability 
##                        0                        0                        0 
##                   energy                      key                 loudness 
##                        0                        0                        0 
##                     mode              speechiness             acousticness 
##                        0                        0                        0 
##         instrumentalness                 liveness                  valence 
##                        0                        0                        0 
##                    tempo              duration_ms track_album_release_date 
##                        0                        0                       31
  • We do have missing values, however we do not need these variables to answer our question.
# Energy Outlier Cleaner
lower_bound_energy <- quantile(spotify_songs$energy, 0.10, na.rm = TRUE)
upper_bound_energy <- quantile(spotify_songs$energy, 0.90, na.rm = TRUE)

spotify_songs$energy[spotify_songs$energy < lower_bound_energy] <- lower_bound_energy
spotify_songs$energy[spotify_songs$energy > upper_bound_energy] <- upper_bound_energy


# Loudness Outlier Cleaner
lower_bound_loudness <- quantile(spotify_songs$loudness, 0.05, na.rm = TRUE)
upper_bound_loudness <- quantile(spotify_songs$loudness, 0.99, na.rm = TRUE)

spotify_songs$loudness[spotify_songs$loudness < lower_bound_loudness] <- lower_bound_loudness
spotify_songs$loudness[spotify_songs$loudness > upper_bound_loudness] <- upper_bound_loudness

# Spilting Data

set.seed(2023)
train_indices <- sample(1:NROW(spotify_songs),NROW(spotify_songs)*0.70)

# Create the training set
train_data <- spotify_songs[train_indices, ]

# Create the testing set
test_data <- spotify_songs[-train_indices, ]

Linear Regression using lm

# Creating the model
lm_model <- lm(track_popularity ~ loudness + valence, data = spotify_songs)
# Summary stats
summary(lm_model)
## 
## Call:
## lm(formula = track_popularity ~ loudness + valence, data = spotify_songs)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -46.497 -18.386   3.028  19.490  58.765 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 44.61767    0.48794  91.440   <2e-16 ***
## loudness     0.57656    0.05219  11.047   <2e-16 ***
## valence      3.27328    0.59058   5.543    3e-08 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 24.92 on 32830 degrees of freedom
## Multiple R-squared:  0.004803,   Adjusted R-squared:  0.004743 
## F-statistic: 79.23 on 2 and 32830 DF,  p-value: < 2.2e-16
# Coefficient stats
coefficients <- coef(lm_model)
print(coefficients)
## (Intercept)    loudness     valence 
##  44.6176721   0.5765613   3.2732823
  • It looks like valence has the strongest correlation with track_popularity

Testing the MSE

lm_track_popularity <- mean((lm_model$fitted.values - spotify_songs$track_popularity)^2)
print(paste("Training MSE for Linear Model:", round(lm_track_popularity, 2)))
## [1] "Training MSE for Linear Model: 621.19"
  • The MSE is high, which indicates a weak correlation for both variables.

KNN using kknn

# KNN modeling wtih k values 5, 10, and 15
knn_model_5 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 5)
summary(knn_model_5)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 5)
## 
## Response: "continuous"
knn_model_10 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 10)
summary(knn_model_10)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 10)
## 
## Response: "continuous"
knn_model_15 <- kknn(track_popularity ~ loudness + valence, train = train_data, test = train_data, k = 15)
summary(knn_model_15)
## 
## Call:
## kknn(formula = track_popularity ~ loudness + valence, train = train_data,     test = train_data, k = 15)
## 
## Response: "continuous"

Testing the MSE

knn_model_5 <- mean((knn_model_5$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 5:", round(knn_model_5, 2)))
## [1] "Training MSE for KNN 5: 282.96"
knn_model_10 <- mean((knn_model_10$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 10:", round(knn_model_10, 2)))
## [1] "Training MSE for KNN 10: 412.26"
knn_model_15 <- mean((knn_model_15$fitted.values - train_data$track_popularity)^2)
print(paste("Training MSE for KNN 15:", round(knn_model_15, 2)))
## [1] "Training MSE for KNN 15: 468.3"

Comparison and Insight + Q&A

KNN MSE

print(paste("Training MSE for KNN 10:", round(knn_model_10, 2)))
## [1] "Training MSE for KNN 10: 412.26"

LM MSE

print(paste("Training MSE for Linear Model:", round(lm_track_popularity, 2)))
## [1] "Training MSE for Linear Model: 621.19"

The KNN model had a better MSE score. The MSE is still not great in terms of correlation, but KNN was still better

The linear model is easier to interpret and is an easier fit for data. But knn has a more flexible approach. The disadvantages are that linear regression will be a poor fit if the data does not form a correlation. KNN is a bit more complex to interpret.

I would recommend KNN since there doesn’t seem to be a correlation between variables.

  1. What variables are most strongly correlated with target? Instrumentalness

  2. How does the value of k in KNN affect the model’s performance (in terms of training MSE and testing MSE)? The MSE seemed to go down the higher the k value went (though not by much).

  3. What assumptions are being made when we use linear regression? Are they met in this dataset? Just describe what you observe from the diagnostic plots. Linear assumptions are: relationship between x and y is linear, variance of residual is the same for any value of x, observations are independent of each other, and for any fixed value of X, Y is normally distributed.

We can tell if the residuals are normally dsitributed through this plot

plot(lm_model, 2)

The plot should ideally have the residuals following in a straight line. In this case, they are not.

  1. Try adding interaction terms to your linear regression model. At least try to find out one interaction term that has a statistically significant coefficient. Report the interaction term and check how these interaction terms influence the model’s performance in terms of R^2 and how do you interpret your new model?
lm_model_interation <- lm(track_popularity ~ speechiness * danceability, data = train_data)
summary(lm_model_interation)
## 
## Call:
## lm(formula = track_popularity ~ speechiness * danceability, data = train_data)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -47.614 -18.329   3.049  19.576  57.080 
## 
## Coefficients:
##                          Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                37.072      1.106  33.505  < 2e-16 ***
## speechiness               -19.821      7.866  -2.520   0.0117 *  
## danceability                8.336      1.693   4.924 8.52e-07 ***
## speechiness:danceability   28.344     11.415   2.483   0.0130 *  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 24.89 on 22979 degrees of freedom
## Multiple R-squared:  0.004637,   Adjusted R-squared:  0.004507 
## F-statistic: 35.69 on 3 and 22979 DF,  p-value: < 2.2e-16

There is less variance explained in the interaction model than the no interaction. I would say this model is less reliable and a worse fit for my data.

  1. Is there any outlier in the dataset? If yes, apply truncation or winsorization techniques to handle outliers. Compare the performance of the models before and after applying these techniques. What differences do you observe?
# Track_popularity does not have outliers

# Danceability 
lower_bound_danceability <- quantile(spotify_songs$danceability, 0.01, na.rm = TRUE)
upper_bound_danceability <- quantile(spotify_songs$danceability, 0.99, na.rm = TRUE)

spotify_songs$danceability[spotify_songs$danceability < lower_bound_danceability] <- lower_bound_danceability
spotify_songs$danceability[spotify_songs$danceability > upper_bound_danceability] <- upper_bound_danceability

# Energy 
lower_bound_energy <- quantile(spotify_songs$energy, 0.10, na.rm = TRUE)
upper_bound_energy <- quantile(spotify_songs$energy, 0.90, na.rm = TRUE)

spotify_songs$energy[spotify_songs$energy < lower_bound_energy] <- lower_bound_energy
spotify_songs$energy[spotify_songs$energy > upper_bound_energy] <- upper_bound_energy


# Loudness 
lower_bound_loudness <- quantile(spotify_songs$loudness, 0.05, na.rm = TRUE)
upper_bound_loudness <- quantile(spotify_songs$loudness, 0.99, na.rm = TRUE)

spotify_songs$loudness[spotify_songs$loudness < lower_bound_loudness] <- lower_bound_loudness
spotify_songs$loudness[spotify_songs$loudness > upper_bound_loudness] <- upper_bound_loudness

# Mode does not have outliers

# Speechiness 
lower_bound_speechiness <- quantile(spotify_songs$speechiness, 0.01, na.rm = TRUE)
upper_bound_speechiness <- quantile(spotify_songs$speechiness, 0.90, na.rm = TRUE)

spotify_songs$speechiness[spotify_songs$speechiness < lower_bound_speechiness] <- lower_bound_speechiness
spotify_songs$speechiness[spotify_songs$speechiness > upper_bound_speechiness] <- upper_bound_speechiness

# Acousticness
lower_bound_acousticness <- quantile(spotify_songs$acousticness, 0.01, na.rm = TRUE)
upper_bound_acousticness <- quantile(spotify_songs$acousticness, 0.90, na.rm = TRUE)

spotify_songs$acousticness[spotify_songs$acousticness < lower_bound_acousticness] <- lower_bound_acousticness
spotify_songs$acousticness[spotify_songs$acousticness > upper_bound_acousticness] <- upper_bound_acousticness

# Instrumentalness
lower_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.01, na.rm = TRUE)
upper_bound_instrumentalness <- quantile(spotify_songs$instrumentalness, 0.75, na.rm = TRUE)

spotify_songs$instrumentalness[spotify_songs$instrumentalness < lower_bound_instrumentalness] <- lower_bound_instrumentalness
spotify_songs$instrumentalness[spotify_songs$instrumentalness > upper_bound_instrumentalness] <- upper_bound_instrumentalness

# Liveness
lower_bound_liveness <- quantile(spotify_songs$liveness, 0.01, na.rm = TRUE)
upper_bound_liveness <- quantile(spotify_songs$liveness, 0.90, na.rm = TRUE)

spotify_songs$liveness[spotify_songs$liveness < lower_bound_liveness] <- lower_bound_liveness
spotify_songs$liveness[spotify_songs$liveness > upper_bound_liveness] <- upper_bound_liveness

# Valence does not have outliers

# Tempo
lower_bound_tempo <- quantile(spotify_songs$tempo, 0.05, na.rm = TRUE)
upper_bound_tempo <- quantile(spotify_songs$tempo, 0.90, na.rm = TRUE)

spotify_songs$tempo[spotify_songs$tempo < lower_bound_tempo] <- lower_bound_tempo
spotify_songs$tempo[spotify_songs$tempo > upper_bound_tempo] <- upper_bound_tempo

# Duration_ms
lower_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.01, na.rm = TRUE)
upper_bound_duration_ms <- quantile(spotify_songs$duration_ms, 0.95, na.rm = TRUE)

spotify_songs$duration_ms[spotify_songs$duration_ms < lower_bound_duration_ms] <- lower_bound_duration_ms
spotify_songs$duration_ms[spotify_songs$duration_ms > upper_bound_duration_ms] <- upper_bound_duration_ms
lm_model_win <- lm(track_popularity ~ danceability, data = spotify_songs)
summary(lm_model_win)
## 
## Call:
## lm(formula = track_popularity ~ danceability, data = spotify_songs)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -45.549 -18.410   2.944  19.475  57.135 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)   35.0478     0.6438   54.44   <2e-16 ***
## danceability  11.3403     0.9600   11.81   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 24.93 on 32831 degrees of freedom
## Multiple R-squared:  0.004233,   Adjusted R-squared:  0.004202 
## F-statistic: 139.6 on 1 and 32831 DF,  p-value: < 2.2e-16

There is slightly more variance explained as indicated by R^2

  1. How could feature scaling (standardization) affect the KNN model? Feature scaling helps prevent dominant numbers from having a greater influence in the KNN model.

  2. What insights can you derive from comparing the linear regression and KNN models? Which model would you recommend the most and Why? I would recommend KNN models for this dataset, since linear regression doesn’t seem to be a true fit. The assumptions are not met.