#Data Exploration
library(MASS) 
library(ggplot2) 
library(corrplot)
## Warning: package 'corrplot' was built under R version 4.1.3
## corrplot 0.92 loaded
library(kknn)
## Warning: package 'kknn' was built under R version 4.1.3

#Motor Trend Car Road Tests ## Data Exploration Analysis Load the data

## The following object is masked from package:ggplot2:
## 
##     mpg

Examine data and variable types

head(mtcars)
##                    mpg cyl disp  hp drat    wt  qsec vs am gear carb
## Mazda RX4         21.0   6  160 110 3.90 2.620 16.46  0  1    4    4
## Mazda RX4 Wag     21.0   6  160 110 3.90 2.875 17.02  0  1    4    4
## Datsun 710        22.8   4  108  93 3.85 2.320 18.61  1  1    4    1
## Hornet 4 Drive    21.4   6  258 110 3.08 3.215 19.44  1  0    3    1
## Hornet Sportabout 18.7   8  360 175 3.15 3.440 17.02  0  0    3    2
## Valiant           18.1   6  225 105 2.76 3.460 20.22  1  0    3    1
str(mtcars)
## 'data.frame':    32 obs. of  11 variables:
##  $ mpg : num  21 21 22.8 21.4 18.7 18.1 14.3 24.4 22.8 19.2 ...
##  $ cyl : num  6 6 4 6 8 6 8 4 4 6 ...
##  $ disp: num  160 160 108 258 360 ...
##  $ hp  : num  110 110 93 110 175 105 245 62 95 123 ...
##  $ drat: num  3.9 3.9 3.85 3.08 3.15 2.76 3.21 3.69 3.92 3.92 ...
##  $ wt  : num  2.62 2.88 2.32 3.21 3.44 ...
##  $ qsec: num  16.5 17 18.6 19.4 17 ...
##  $ vs  : num  0 0 1 1 0 1 0 1 1 1 ...
##  $ am  : num  1 1 1 0 0 0 0 0 0 0 ...
##  $ gear: num  4 4 4 3 3 3 3 4 4 4 ...
##  $ carb: num  4 4 1 1 2 1 4 2 2 4 ...

Summary statistics

summary(mtcars)
##       mpg             cyl             disp             hp       
##  Min.   :10.40   Min.   :4.000   Min.   : 71.1   Min.   : 52.0  
##  1st Qu.:15.43   1st Qu.:4.000   1st Qu.:120.8   1st Qu.: 96.5  
##  Median :19.20   Median :6.000   Median :196.3   Median :123.0  
##  Mean   :20.09   Mean   :6.188   Mean   :230.7   Mean   :146.7  
##  3rd Qu.:22.80   3rd Qu.:8.000   3rd Qu.:326.0   3rd Qu.:180.0  
##  Max.   :33.90   Max.   :8.000   Max.   :472.0   Max.   :335.0  
##       drat             wt             qsec             vs        
##  Min.   :2.760   Min.   :1.513   Min.   :14.50   Min.   :0.0000  
##  1st Qu.:3.080   1st Qu.:2.581   1st Qu.:16.89   1st Qu.:0.0000  
##  Median :3.695   Median :3.325   Median :17.71   Median :0.0000  
##  Mean   :3.597   Mean   :3.217   Mean   :17.85   Mean   :0.4375  
##  3rd Qu.:3.920   3rd Qu.:3.610   3rd Qu.:18.90   3rd Qu.:1.0000  
##  Max.   :4.930   Max.   :5.424   Max.   :22.90   Max.   :1.0000  
##        am              gear            carb      
##  Min.   :0.0000   Min.   :3.000   Min.   :1.000  
##  1st Qu.:0.0000   1st Qu.:3.000   1st Qu.:2.000  
##  Median :0.0000   Median :4.000   Median :2.000  
##  Mean   :0.4062   Mean   :3.688   Mean   :2.812  
##  3rd Qu.:1.0000   3rd Qu.:4.000   3rd Qu.:4.000  
##  Max.   :1.0000   Max.   :5.000   Max.   :8.000
table(mtcars$mpg)
## 
## 10.4 13.3 14.3 14.7   15 15.2 15.5 15.8 16.4 17.3 17.8 18.1 18.7 19.2 19.7   21 
##    2    1    1    1    1    2    1    1    1    1    1    1    1    2    1    2 
## 21.4 21.5 22.8 24.4   26 27.3 30.4 32.4 33.9 
##    2    1    2    1    1    1    2    1    1

QUESTION 1. What variables are most strongly correlated with mpg?

The variables more strongly correlated with the variable are wt(weigth), hp (horse power), and cyl (number of cylinders), tho this three are negatively correlated. This suggests that the higher the value of them, the less miles per gallon the cars got.

cor(mtcars$mpg, mtcars[, -1])
##            cyl       disp         hp      drat         wt     qsec        vs
## [1,] -0.852162 -0.8475514 -0.7761684 0.6811719 -0.8676594 0.418684 0.6640389
##             am      gear       carb
## [1,] 0.5998324 0.4802848 -0.5509251

Some visualization

ggplot(mtcars, aes(x=mpg)) +
  geom_histogram(binwidth=5, fill="blue", alpha=0.5) +
  ggtitle("Distribution Median of Miles/Gallon") +
  xlab("Median value of mi/gal in the Motor Trend Car Road Test") +
  ylab("Frequency")

ggplot(mtcars, aes(x = hp , y = mpg)) +
  geom_point(aes(color = cyl)) +
  ggtitle("Vehicle Horse Power vs.Miles per Gallon") +
  xlab("Horse Power") +
  ylab("Miles per gallon ")

ggplot(mtcars, aes(x = wt, y = mpg)) +
  geom_boxplot(aes(fill=factor(cyl))) +
  ggtitle("Median miles per gallon to weight") +
  xlab("Vehicle weight") +
  ylab("Median miles per gallon")

### Correlation Matrix visualization

corr_matrix <- cor(mtcars)
print(corr_matrix)
##             mpg        cyl       disp         hp        drat         wt
## mpg   1.0000000 -0.8521620 -0.8475514 -0.7761684  0.68117191 -0.8676594
## cyl  -0.8521620  1.0000000  0.9020329  0.8324475 -0.69993811  0.7824958
## disp -0.8475514  0.9020329  1.0000000  0.7909486 -0.71021393  0.8879799
## hp   -0.7761684  0.8324475  0.7909486  1.0000000 -0.44875912  0.6587479
## drat  0.6811719 -0.6999381 -0.7102139 -0.4487591  1.00000000 -0.7124406
## wt   -0.8676594  0.7824958  0.8879799  0.6587479 -0.71244065  1.0000000
## qsec  0.4186840 -0.5912421 -0.4336979 -0.7082234  0.09120476 -0.1747159
## vs    0.6640389 -0.8108118 -0.7104159 -0.7230967  0.44027846 -0.5549157
## am    0.5998324 -0.5226070 -0.5912270 -0.2432043  0.71271113 -0.6924953
## gear  0.4802848 -0.4926866 -0.5555692 -0.1257043  0.69961013 -0.5832870
## carb -0.5509251  0.5269883  0.3949769  0.7498125 -0.09078980  0.4276059
##             qsec         vs          am       gear        carb
## mpg   0.41868403  0.6640389  0.59983243  0.4802848 -0.55092507
## cyl  -0.59124207 -0.8108118 -0.52260705 -0.4926866  0.52698829
## disp -0.43369788 -0.7104159 -0.59122704 -0.5555692  0.39497686
## hp   -0.70822339 -0.7230967 -0.24320426 -0.1257043  0.74981247
## drat  0.09120476  0.4402785  0.71271113  0.6996101 -0.09078980
## wt   -0.17471588 -0.5549157 -0.69249526 -0.5832870  0.42760594
## qsec  1.00000000  0.7445354 -0.22986086 -0.2126822 -0.65624923
## vs    0.74453544  1.0000000  0.16834512  0.2060233 -0.56960714
## am   -0.22986086  0.1683451  1.00000000  0.7940588  0.05753435
## gear -0.21268223  0.2060233  0.79405876  1.0000000  0.27407284
## carb -0.65624923 -0.5696071  0.05753435  0.2740728  1.00000000
corrplot(corr_matrix, method="circle", type="upper", order="hclust",
         tl.col="black", tl.srt=45)

Data Preprocessing

The data may contain some missing values or outliers that might affect the result, let’s check the ### Check whether there are any missing Values for each column

colSums(is.na(Boston))
##    crim      zn   indus    chas     nox      rm     age     dis     rad     tax 
##       0       0       0       0       0       0       0       0       0       0 
## ptratio   black   lstat    medv 
##       0       0       0       0

Check whether there are any outliers for the values in each column

boxplot(mtcars, las=2)

#### QUESTION 5. Is there any outliers in the dataset? If yes, apply truncation or winsorization techniques to handle outliers. Compare the performance of the models before and after applying these techniques. What differences do you observe? There are significant changes, the min value increades, the max value decreased,rest of the values stayed the same, except for the mean. The mean decreased the most because the biggest outlier was a very vehicle with a very high hp.

upper_bound_hp <- quantile(mtcars$hp, 0.99, na.rm = TRUE)
lower_bound_hp <- quantile(mtcars$hp, 0.01, na.rm = TRUE)
mtcars$hp[mtcars$hp > upper_bound_hp] <- upper_bound_hp
mtcars$hp[mtcars$hp < lower_bound_hp] <- lower_bound_hp
summary(mtcars[c("hp")])
##        hp       
##  Min.   : 55.1  
##  1st Qu.: 96.5  
##  Median :123.0  
##  Mean   :146.1  
##  3rd Qu.:180.0  
##  Max.   :313.0

3.Linear Regression using lm

Linear regression

mtcars_lm <- lm(mpg ~ ., data = mtcars)
summary(mtcars_lm)
## 
## Call:
## lm(formula = mpg ~ ., data = mtcars)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -3.4201 -1.5602 -0.1017  1.1611  4.6188 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)  
## (Intercept) 12.72098   18.68259   0.681   0.5034  
## cyl         -0.09268    1.04285  -0.089   0.9300  
## disp         0.01389    0.01778   0.782   0.4432  
## hp          -0.02414    0.02288  -1.055   0.3034  
## drat         0.86325    1.62519   0.531   0.6009  
## wt          -3.68497    1.87561  -1.965   0.0628 .
## qsec         0.78554    0.73222   1.073   0.2955  
## vs           0.32962    2.09148   0.158   0.8763  
## am           2.46214    2.04818   1.202   0.2427  
## gear         0.65888    1.48812   0.443   0.6625  
## carb        -0.20609    0.80918  -0.255   0.8014  
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 2.642 on 21 degrees of freedom
## Multiple R-squared:  0.8698, Adjusted R-squared:  0.8079 
## F-statistic: 14.03 on 10 and 21 DF,  p-value: 3.562e-07

####QUESTION 4. Try adding interaction terms to your linear regression model. At least try to find out one interaction term that has a statistically significant coefficient. Report the interaction term and check how do these interaction terms influence the model’s performance in terms of R^2 and how do you interpret your new model? The model has a statiscally significant coefficient because p<0.05. Yet, in terms ogf R², the first linear regession is highers, which indicates a better fit.

mtcars_lm_2 <- lm(mpg ~ vs + hp + vs*hp, data = mtcars)
summary(mtcars_lm_2)
## 
## Call:
## lm(formula = mpg ~ vs + hp + vs * hp, data = mtcars)
## 
## Residuals:
##    Min     1Q Median     3Q    Max 
## -5.469 -1.733 -0.254  1.597  9.284 
## 
## Coefficients:
##             Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 25.15796    2.81567   8.935 1.09e-09 ***
## vs          14.10964    4.63952   3.041  0.00507 ** 
## hp          -0.04531    0.01433  -3.163  0.00374 ** 
## vs:hp       -0.11532    0.04158  -2.773  0.00976 ** 
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 3.385 on 28 degrees of freedom
## Multiple R-squared:  0.7151, Adjusted R-squared:  0.6846 
## F-statistic: 23.42 on 3 and 28 DF,  p-value: 8.627e-08
comparison_df <- data.frame(Actual = mtcars_lm$fitted.values, lm_predicted = mtcars$mpg)
head(comparison_df)
##                     Actual lm_predicted
## Mazda RX4         22.64764         21.0
## Mazda RX4 Wag     22.14787         21.0
## Datsun 710        26.22011         22.8
## Hornet 4 Drive    21.27643         21.4
## Hornet Sportabout 17.73353         18.7
## Valiant           20.37236         18.1

QUESTION3. What assumptions are being made when we use linear regression? Are they met in this dataset? Just describe what you observe from the diagnostic plots.

Based on the graph of “Residuals vs fitted” the linearity condition is met, all the values are randomly distributed around a horizontal line at 0. The “theoretical qantiles” tend to a diagonal line and this means it tends to normal. So, the assumptions are being met.

par(mfrow = c(2, 2))
plot(mtcars_lm)

Evaluate the model using mean squared error

lm_mse <- mean((mtcars_lm$fitted.values - mtcars$mpg)^2)
print(paste("Mean Squared Error for Linear Model:", round(lm_mse, 2)))
## [1] "Mean Squared Error for Linear Model: 4.58"

KNN using kknn

For k=5

k <- 5 
knn_fit_std <- kknn(mpg ~ ., mtcars, mtcars, k=k)

Manually check the results

comparison_df$knn_std_predicted <- knn_fit_std$fitted.values
head(comparison_df)
##                     Actual lm_predicted knn_std_predicted
## Mazda RX4         22.64764         21.0          21.22292
## Mazda RX4 Wag     22.14787         21.0          21.22292
## Datsun 710        26.22011         22.8          25.63612
## Hornet 4 Drive    21.27643         21.4          20.65088
## Hornet Sportabout 17.73353         18.7          17.98029
## Valiant           20.37236         18.1          19.93957

Evaluate the model

knn_std_mse <- mean((knn_fit_std$fitted.values - mtcars$mpg)^2)
print(paste("Mean Squared Error for stdKNN:", round(knn_std_mse, 2)))
## [1] "Mean Squared Error for stdKNN: 2.11"

Try different values of k

mse_df <- data.frame(k = integer(), MSE = numeric())

for (k in c(1, 3, 5, 7, 9, 11)) {
  knn_model <- kknn(mpg ~ ., train = mtcars, test = mtcars, k = k)

  mse <-  mean((knn_model$fitted.values - mtcars$mpg)^2)
  mse_df <- rbind(mse_df, data.frame(k = k, MSE = mse))
}

print(mse_df)
##    k       MSE
## 1  1 0.0000000
## 2  3 0.8010385
## 3  5 2.1134289
## 4  7 3.3578251
## 5  9 4.2663137
## 6 11 4.9237622

QUESTION 2. How does the value of k in KNN affect the model’s performance?

The smaller the value it appeared to have a better fit. With 1 having no mean square error, but this also means that it will only fit this specific data sets. An accurate prediction can not be that small because the smaller the value of k data is more suceptible to the noise. As well, k can not be too large because it will be biased towards the majority class.

Comparison and Insights

Compare the performance of the linear regression model and the KNN model.

The data suited both models pretty well, but it fitted the best the KNN model with a k value of 5. As previously discussed, the value of k should not me nor too high nor too low, so I chose 5.

Discuss the advantages and disadvantages of both methods.

Linear regression is very easy to understand and very reliable if the relationships are linear, but for the same reason, it is susceptible to outliers. KNN on the other hand, is more flexible because it can capture non-linear relationships, thus making it harder to understand. #### Explain which model you would recommend and why. I think that KNN suits the problem the best, the data predicted by KNN was more accurate in the table that portrayed both. And to reinforce my belief, after conducting a least mean squared test for both of them I realized KNN’S result was way lower than the value for linear regression.

####6. How could feature scaling affect the KNN model? KNN is a distance based model. Feature scaling ensures that all the features contribute equally to the calculation of distances in the model.

####7. What insights can you derive from comparing the linear regression and KNN models? The thing that I understood the best is that both prediction models are good and efficient, but to determine which one to use and how to interpret the results the proper calculations have to be made and choose the one with the lowest mean squared error.