################################################################################
######## NAME: NIVEDITHA KM ####################################################
######## ASSIGNMENT 3 - DATA MINING#############################################
################################################################################
library(tidyverse) # data manipulation
library(ggpubr)
################################################################################
#DATA SETS IMPORT
#DATA PRE-PROCESSING STARTS
################################################################################
test_df <- read.csv("test.csv")
train_df <- read.csv("train.csv")
#summary(test_df)
tst_df <- test_df %>% drop_na()
trn_df <- train_df %>% drop_na()
#view(trn_df)
#view(tst_df)
#FOLLOWING PIECE OF CODE IS TO REMOVE COMMAS FROM COLUMNS IF ANY(Not necessary for this though)
tst_df <- data.frame(lapply(tst_df, function(x) {
gsub(",", "", x)
}))
trn_df <- data.frame(lapply(trn_df, function(x) {
gsub(",", "", x)
}))
#################################################################################
#FOLLOWING PIECE OF CODE IS TO CHECK IF NUMERIC CONVERSION CAN HAPPEN ON COLUMN
#THIS IS DONE ON BOTH TEST AND TRAINING DATASET
#USUALLY FULL DATASET IS BROKEN INTO TEST AND TRAINING. IN THIS CASE THESE WERE
#PROVIDED SEPARATELY WHICH WAS HELPFUL.
#################################################################################
is_all_numeric <- function(x) {
!any(is.na(suppressWarnings(as.numeric(na.omit(x))))) & is.character(x)
}
tst_df %>%
mutate_if(is_all_numeric,as.numeric) %>%
str()
## 'data.frame': 300 obs. of 2 variables:
## $ x: num 77 21 22 20 36 15 62 95 20 5 ...
## $ y: num 79.8 23.2 25.6 17.9 41.8 ...
trn_df %>%
mutate_if(is_all_numeric,as.numeric) %>%
str()
## 'data.frame': 699 obs. of 2 variables:
## $ x: num 24 50 15 38 87 36 12 81 25 5 ...
## $ y: num 21.5 47.5 17.2 36.6 87.3 ...
# CONVERTS CHARACTER COLUMNS TO NUMERIC COLUMNS
tst_df <- tst_df %>% mutate_if(is_all_numeric,as.numeric)
trn_df <- trn_df %>% mutate_if(is_all_numeric,as.numeric)
print('TEST DATASET SUMMARY')
## [1] "TEST DATASET SUMMARY"
summary(test_df)
## x y
## Min. : 0.00 Min. : -3.468
## 1st Qu.: 27.00 1st Qu.: 25.677
## Median : 53.00 Median : 52.171
## Mean : 50.94 Mean : 51.205
## 3rd Qu.: 73.00 3rd Qu.: 74.303
## Max. :100.00 Max. :105.592
print('TRAINING DATASET SUMMARY')
## [1] "TRAINING DATASET SUMMARY"
summary(trn_df)
## x y
## Min. : 0.00 Min. : -3.84
## 1st Qu.: 25.00 1st Qu.: 24.93
## Median : 49.00 Median : 48.97
## Mean : 50.01 Mean : 49.94
## 3rd Qu.: 75.00 3rd Qu.: 74.93
## Max. :100.00 Max. :108.87
################################################################################
#DATA PRE-PROCESSING ENDS
################################################################################
#################################################################################
#DATA EXPLORATION OF THE TEST AND TRAINING DATASET IS DONE BELOW #
#################################################################################
#################################################################################
# SCATTERPLOT OF TEST AND TRAINING DATASET SHOWING OF DATA LINEARITY #
#################################################################################
scatter.smooth(x=tst_df$x, y=tst_df$y, main="Test Data y ~ x",xlab = 'X axis', ylab = 'Y axis')
scatter.smooth(x=trn_df$x, y=trn_df$y, main="Train Data y ~ x",xlab = 'X axis', ylab = 'Y axis')
#################################################################################
# BOX PLOT TO SEE IF THERE ARE ANY OUTLIERS IN DATASETS #
#################################################################################
par(mfrow=c(1, 2)) # divide graph area in 2 columns
boxplot(tst_df$x, main="x column", sub=paste("Outlier rows: ", boxplot.stats(tst_df$x)$out)) # box plot for 'speed'
boxplot(tst_df$y, main="y column", sub=paste("Outlier rows: ", boxplot.stats(tst_df$y)$out)) # box plot for 'distance'
par(mfrow=c(1, 2)) # divide graph area in 2 columns
boxplot(trn_df$x, main="x column", sub=paste("Outlier rows: ", boxplot.stats(trn_df$x)$out)) # box plot for 'speed'
boxplot(trn_df$y, main="y column", sub=paste("Outlier rows: ", boxplot.stats(trn_df$y)$out)) # box plot for 'distance'
#################################################################################
# DENSITY PLOT TO SHOW PREDICTOR VARIABLE IS DISTRIBUTED NORMALLY (BELL CURVE) #
#################################################################################
#A small bandwidth leads to undersmoothing. Density plot will have lot of individual peeks. While huge bandwidth leads to oversmoothing plot will look like unimodal distribution. Which is how the Density plots of both Test and Train looks like.
library(e1071)
par(mfrow=c(1, 2)) # divide graph area in 2 columns
plot(density(trn_df$x), main="Density Plot: x Train Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(trn_df$x), 2))) # density plot for 'speed'
polygon(density(trn_df$x), col="red")
plot(density(trn_df$y), main="Density Plot: y Train Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(trn_df$y), 2))) # density plot for 'y'
polygon(density(trn_df$y), col="red")
library(e1071)
par(mfrow=c(1, 2)) # divide graph area in 2 columns
plot(density(tst_df$x), main="Density Plot: x Test Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(tst_df$x), 2))) # density plot for 'speed'
polygon(density(tst_df$x), col="red")
plot(density(tst_df$y), main="Density Plot: y Test Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(tst_df$y), 2))) # density plot for 'y'
polygon(density(tst_df$y), col="red")
#################################################################################
# CORRELATION #
#################################################################################
#Correlation confirms how close two variables are related. We can see the values are close to 1 hence both x and y of Training and Test dataset are closely related.
cor(tst_df$x, tst_df$y)
## [1] 0.9945453
cor(trn_df$x, trn_df$y)
## [1] 0.9953399
#################################################################################
# NORMALITY CHECK OF DATA CONFIRMS PREDICTOR DATA IS NORMALLY DISTRIBUTED #
#################################################################################
hist(trn_df$y)
hist(tst_df$y)
####################################################################
#Plot to show the linear model is perfect model for this data
#####################################################################
ggplot(data = trn_df, aes(x = x, y = y)) +
geom_point() +
stat_smooth(method = "lm", col = "dodgerblue3") +
theme(panel.background = element_rect(fill = "white"),
axis.line.x=element_line(),
axis.line.y=element_line()) +
ggtitle("Linear Model Fitted to Data")
## `geom_smooth()` using formula = 'y ~ x'
###################################################################
# APPLYING LINEAR MODEL TO TRAINING DATA
###################################################################
model = lm(y~x, data=trn_df)
summary(model)
##
## Call:
## lm(formula = y ~ x, data = trn_df)
##
## Residuals:
## Min 1Q Median 3Q Max
## -9.1523 -2.0179 0.0325 1.8573 8.9132
##
## Coefficients:
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) -0.107265 0.212170 -0.506 0.613
## x 1.000656 0.003672 272.510 <2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 2.809 on 697 degrees of freedom
## Multiple R-squared: 0.9907, Adjusted R-squared: 0.9907
## F-statistic: 7.426e+04 on 1 and 697 DF, p-value: < 2.2e-16
par(mfrow=c(2,2))
plot(model)
#############################################################################
# BELOW COMMENTS AND VALIDATIONS SHOW WHY THIS MODEL IS GOOD FOR THIS DATA
#############################################################################
# F-statistic signifies it can be found that there is a close relationship between predictor and response variable. Here the F Statistic is 7.426e+04 this far more than 1 which means there is inter relation between predictor x and response y.
# Lesser the P value more important the predictor variable. The preditor variable is 2.2e-16 which is far lesser than 0.
# The model is fit if the R-squared(R^2) value is closer to 1. R-squared value for this model is 0.9907. So model is perfect fit
# From Residuals vs Fitted graph it can be seen the red trend line is driving through zero.
# QQ Residuals plot is good if the points are close or on the line. We can see that it is.
coeff<- summary(model)$coefficients
beta.estimate <- coeff["x", "Estimate"]
std.error <- coeff["x", "Std. Error"]
t_value <- beta.estimate/std.error
p_value <- 2*pt(-abs(t_value), df=nrow(trn_df)-ncol(trn_df))
#anova(model)
f_statistic <- model$fstatistic[1] # fstatistic
f <- summary(model)$fstatistic
model_p <- pf(f[1], f[2], f[3], lower=FALSE)
print('pvalue, tvalue fvalue')
## [1] "pvalue, tvalue fvalue"
paste("t-Value of training model", t_value)
## [1] "t-Value of training model 272.509786594665"
paste("p-VALUE of training model" , p_value)
## [1] "p-VALUE of training model 0"
paste("F-Statistic of training model", f)
## [1] "F-Statistic of training model 74261.5837898699"
## [2] "F-Statistic of training model 1"
## [3] "F-Statistic of training model 697"
paste("MODEL P Value of training model", model_p)
## [1] "MODEL P Value of training model 0"
##################################################################################
# Residual is a measure of how far away a point is vertically from the regression
# line. Simply, it is the error between a predicted value and the observed actual
# value. Here residuals form a bell curve which means residual is normally
# distributed
# This proves that the Linear Regression model fits training data well.
##################################################################################
ggplot(data=trn_df, aes(model$residuals)) +
geom_histogram(binwidth = 1, color = "black", fill = "purple4") +
theme(panel.background = element_rect(fill = "white"),
axis.line.x=element_line(),
axis.line.y=element_line()) +
ggtitle("Histogram for Model Residuals on Training Data")
#################################################################################
#APPLYING MODEL TO GET PREDICTION ON TEST DATA
#################################################################################
tst_df$pred <- predict(model, newdata = tst_df)
#trn_df$pred <- predict(model, newdata = trn_df)
#view(tst_df)
#view(trn_df)
#################################################################################
# RMSE SHOWS HOW FAR THE PREDICTED VALUE IS FROM EXPECTED VALUE
# IN THIS CASE IT IS AROUND 3 SO IT IS GOOD MODEL
#################################################################################
rmse_tst <- sqrt(sum(((tst_df$pred) - tst_df$y)^2)/length(tst_df$y))
#rmse_trn <- sqrt(sum(((trn_df$pred) - trn_df$y)^2)/length(trn_df$y))
c(RMSE = rmse_tst, R2=summary(model)$r.squared)
## RMSE R2
## 3.0713063 0.9907015
#c(RMSE = rmse_trn, R2=summary(model)$r.squared)
par(mfrow=c(1,1))
##################################################################################
# THE SCATTER PLOT OF EXPECTED VALUES OF TEST AND PREDICTED VALUES OF TEST ARE
# PLOTTED
##################################################################################
plot(tst_df$x, tst_df$y,main="Scatter plot of x vs y values", ylab="Expected Values of Test",xlab="X Values of Test")
plot(tst_df$x, tst_df$pred,main="Scatter plot of x vs predicted y values", ylab="Predicted Values of Test",xlab="X Values of Test")
#plot(trn_df$y, trn_df$pred)
#################################################################################
# THE SCATTER PLOT OF EXPECTED VALUES OF TEST AND PREDICTED VALUES OF TEST ARE
# PLOTTED USING GGPLOT
#################################################################################
library(ggplot2)
# Basic scatter plot
#ggplot(tst_df, aes(x=x, y=y)) + geom_point()
# Change the point size, and shape
#ggplot(tst_df, aes(x=x, y=pred)) +
# geom_point()
ggplot(tst_df, aes(x=x, y=y)) +
geom_point(aes(size=2))
ggplot(tst_df, aes(x=x, y=pred)) +
geom_point(aes(size=2))
F-statistic signifies it can be found that there is a close relationship between predictor and response variable. Here the F Statistic is 7.426e+04 this far more than 1 which means there is inter relation between predictor x and response y.
Lesser the P value more important the predictor variable. The preditor variable is 2.2e-16 which is far lesser than 0.
The model is fit if the R-squared(R^2) value is closer to 1. R-squared value for this model is 0.9907. So model is perfect fit.
From Residuals vs Fitted graph it can be seen the red trend line is driving through zero.
QQ Residuals plot is good if the points are close or on the line. We can see that it is.