CODE BLOCK FOR DATA PREPARATION

################################################################################
######## NAME: NIVEDITHA KM ####################################################
######## ASSIGNMENT 3 - DATA MINING#############################################
################################################################################

library(tidyverse)  # data manipulation
library(ggpubr)
################################################################################
#DATA SETS IMPORT
#DATA PRE-PROCESSING STARTS
################################################################################

test_df <- read.csv("test.csv")
train_df <- read.csv("train.csv")

#summary(test_df)

tst_df <- test_df %>% drop_na()
trn_df <- train_df %>% drop_na()

#view(trn_df)
#view(tst_df)


#FOLLOWING PIECE OF CODE IS TO REMOVE COMMAS FROM COLUMNS IF ANY(Not necessary for this though)
tst_df <- data.frame(lapply(tst_df, function(x) {
  gsub(",", "", x)
  }))

trn_df <- data.frame(lapply(trn_df, function(x) {
  gsub(",", "", x)
  }))

#################################################################################
#FOLLOWING PIECE OF CODE IS TO CHECK IF NUMERIC CONVERSION CAN HAPPEN ON COLUMN 
#THIS IS DONE ON BOTH TEST AND TRAINING DATASET
#USUALLY FULL DATASET IS BROKEN INTO TEST AND TRAINING. IN THIS CASE THESE WERE
#PROVIDED SEPARATELY WHICH WAS HELPFUL.
#################################################################################

is_all_numeric <- function(x) {
  !any(is.na(suppressWarnings(as.numeric(na.omit(x))))) & is.character(x)
}

tst_df %>% 
  mutate_if(is_all_numeric,as.numeric) %>%
  str()
## 'data.frame':    300 obs. of  2 variables:
##  $ x: num  77 21 22 20 36 15 62 95 20 5 ...
##  $ y: num  79.8 23.2 25.6 17.9 41.8 ...
trn_df %>% 
  mutate_if(is_all_numeric,as.numeric) %>%
  str()
## 'data.frame':    699 obs. of  2 variables:
##  $ x: num  24 50 15 38 87 36 12 81 25 5 ...
##  $ y: num  21.5 47.5 17.2 36.6 87.3 ...
# CONVERTS CHARACTER COLUMNS TO NUMERIC COLUMNS
tst_df <- tst_df %>% mutate_if(is_all_numeric,as.numeric)

trn_df <- trn_df %>% mutate_if(is_all_numeric,as.numeric)

print('TEST DATASET SUMMARY')
## [1] "TEST DATASET SUMMARY"
summary(test_df)
##        x                y          
##  Min.   :  0.00   Min.   : -3.468  
##  1st Qu.: 27.00   1st Qu.: 25.677  
##  Median : 53.00   Median : 52.171  
##  Mean   : 50.94   Mean   : 51.205  
##  3rd Qu.: 73.00   3rd Qu.: 74.303  
##  Max.   :100.00   Max.   :105.592
print('TRAINING DATASET SUMMARY')
## [1] "TRAINING DATASET SUMMARY"
summary(trn_df)
##        x                y         
##  Min.   :  0.00   Min.   : -3.84  
##  1st Qu.: 25.00   1st Qu.: 24.93  
##  Median : 49.00   Median : 48.97  
##  Mean   : 50.01   Mean   : 49.94  
##  3rd Qu.: 75.00   3rd Qu.: 74.93  
##  Max.   :100.00   Max.   :108.87
################################################################################
#DATA PRE-PROCESSING ENDS
################################################################################

DATA EXPLORATION

#################################################################################
#DATA EXPLORATION OF THE TEST AND TRAINING DATASET IS DONE BELOW                #
#################################################################################

#################################################################################
# SCATTERPLOT OF TEST AND TRAINING DATASET SHOWING OF DATA LINEARITY            #
#################################################################################

scatter.smooth(x=tst_df$x, y=tst_df$y, main="Test Data y ~ x",xlab = 'X axis', ylab = 'Y axis')

scatter.smooth(x=trn_df$x, y=trn_df$y, main="Train Data y ~ x",xlab = 'X axis', ylab = 'Y axis')  

#################################################################################
#              BOX PLOT TO SEE IF THERE ARE ANY OUTLIERS IN DATASETS            #
#################################################################################

par(mfrow=c(1, 2))  # divide graph area in 2 columns 
boxplot(tst_df$x, main="x column", sub=paste("Outlier rows: ", boxplot.stats(tst_df$x)$out))  # box plot for 'speed'
boxplot(tst_df$y, main="y column", sub=paste("Outlier rows: ", boxplot.stats(tst_df$y)$out))  # box plot for 'distance'

par(mfrow=c(1, 2))  # divide graph area in 2 columns
boxplot(trn_df$x, main="x column", sub=paste("Outlier rows: ", boxplot.stats(trn_df$x)$out))  # box plot for 'speed'
boxplot(trn_df$y, main="y column", sub=paste("Outlier rows: ", boxplot.stats(trn_df$y)$out))  # box plot for 'distance'

#################################################################################
# DENSITY PLOT TO SHOW PREDICTOR VARIABLE IS DISTRIBUTED NORMALLY (BELL CURVE)  #
#################################################################################


#A small bandwidth leads to undersmoothing. Density plot will have lot of individual peeks. While huge bandwidth leads to oversmoothing plot will look like unimodal distribution. Which is how the Density plots of both Test and Train looks like.

library(e1071)
par(mfrow=c(1, 2))  # divide graph area in 2 columns
plot(density(trn_df$x), main="Density Plot: x Train Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(trn_df$x), 2)))  # density plot for 'speed'
polygon(density(trn_df$x), col="red")
plot(density(trn_df$y), main="Density Plot: y Train Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(trn_df$y), 2)))  # density plot for 'y'
polygon(density(trn_df$y), col="red")

library(e1071)
par(mfrow=c(1, 2))  # divide graph area in 2 columns
plot(density(tst_df$x), main="Density Plot: x Test Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(tst_df$x), 2)))  # density plot for 'speed'
polygon(density(tst_df$x), col="red")
plot(density(tst_df$y), main="Density Plot: y Test Dataset", ylab="Frequency", sub=paste("Skewness:", round(e1071::skewness(tst_df$y), 2)))  # density plot for 'y'
polygon(density(tst_df$y), col="red")

#################################################################################
#                                  CORRELATION                                  #
#################################################################################


#Correlation confirms how close two variables are related. We can see the values are close to 1 hence both x and y of Training and Test dataset are closely related.

cor(tst_df$x, tst_df$y)
## [1] 0.9945453
cor(trn_df$x, trn_df$y)
## [1] 0.9953399
#################################################################################
#    NORMALITY CHECK OF DATA CONFIRMS PREDICTOR DATA IS NORMALLY DISTRIBUTED    #
#################################################################################

hist(trn_df$y)
hist(tst_df$y)

LINEAR REGRESSION MODEL AND ANALYSIS

####################################################################
#Plot to show the linear model is perfect model for this data
#####################################################################
ggplot(data = trn_df, aes(x = x, y = y)) +
geom_point() +
stat_smooth(method = "lm", col = "dodgerblue3") +
theme(panel.background = element_rect(fill = "white"),
axis.line.x=element_line(),
axis.line.y=element_line()) +
ggtitle("Linear Model Fitted to Data")
## `geom_smooth()` using formula = 'y ~ x'

###################################################################
# APPLYING LINEAR MODEL TO TRAINING DATA        
###################################################################

model = lm(y~x, data=trn_df)
summary(model)
## 
## Call:
## lm(formula = y ~ x, data = trn_df)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -9.1523 -2.0179  0.0325  1.8573  8.9132 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept) -0.107265   0.212170  -0.506    0.613    
## x            1.000656   0.003672 272.510   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 2.809 on 697 degrees of freedom
## Multiple R-squared:  0.9907, Adjusted R-squared:  0.9907 
## F-statistic: 7.426e+04 on 1 and 697 DF,  p-value: < 2.2e-16
par(mfrow=c(2,2))
plot(model)

#############################################################################
# BELOW COMMENTS AND VALIDATIONS SHOW WHY THIS MODEL IS GOOD FOR THIS DATA
#############################################################################


# F-statistic signifies it can be found that there is a close relationship between predictor and response variable. Here the F Statistic is 7.426e+04 this far more than 1 which means there is inter relation between predictor x and response y.

# Lesser the P value more important the predictor variable. The preditor variable is 2.2e-16 which is far lesser than 0.

# The model is fit if the R-squared(R^2) value is closer to 1. R-squared value for this model is 0.9907. So model is perfect fit

# From Residuals vs Fitted graph it can be seen the red trend line is driving through zero.

# QQ Residuals plot is good if the points are close or on the line. We can see that it is.


coeff<- summary(model)$coefficients
beta.estimate <- coeff["x", "Estimate"] 
std.error <- coeff["x", "Std. Error"]
t_value <- beta.estimate/std.error
p_value <- 2*pt(-abs(t_value), df=nrow(trn_df)-ncol(trn_df))
#anova(model)
f_statistic <- model$fstatistic[1]  # fstatistic
f <- summary(model)$fstatistic
model_p <- pf(f[1], f[2], f[3], lower=FALSE)

print('pvalue, tvalue fvalue')
## [1] "pvalue, tvalue fvalue"
paste("t-Value of training model", t_value)
## [1] "t-Value of training model 272.509786594665"
paste("p-VALUE of training model" , p_value)
## [1] "p-VALUE of training model 0"
paste("F-Statistic of training model", f)
## [1] "F-Statistic of training model 74261.5837898699"
## [2] "F-Statistic of training model 1"               
## [3] "F-Statistic of training model 697"
paste("MODEL P Value of training model", model_p)
## [1] "MODEL P Value of training model 0"
##################################################################################
# Residual is a measure of how far away a point is vertically from the regression
# line. Simply, it is the error between a predicted value and the observed actual
# value. Here residuals form a bell curve which means residual is normally
# distributed
# This proves that the Linear Regression model fits training data well.
##################################################################################

ggplot(data=trn_df, aes(model$residuals)) +
geom_histogram(binwidth = 1, color = "black", fill = "purple4") +
theme(panel.background = element_rect(fill = "white"),
axis.line.x=element_line(),
axis.line.y=element_line()) +
ggtitle("Histogram for Model Residuals on Training Data")

USING LINEAR MODEL TO PREDICT VALUES ON TEST DATA

#################################################################################
#APPLYING MODEL TO GET PREDICTION ON TEST DATA
#################################################################################
tst_df$pred <- predict(model, newdata = tst_df)
#trn_df$pred <- predict(model, newdata = trn_df)

#view(tst_df)
#view(trn_df)

#################################################################################
# RMSE SHOWS HOW FAR THE PREDICTED VALUE IS FROM EXPECTED VALUE
# IN THIS CASE IT IS AROUND 3 SO IT IS GOOD MODEL
#################################################################################

rmse_tst <- sqrt(sum(((tst_df$pred) - tst_df$y)^2)/length(tst_df$y))

#rmse_trn <- sqrt(sum(((trn_df$pred) - trn_df$y)^2)/length(trn_df$y))

c(RMSE = rmse_tst, R2=summary(model)$r.squared)
##      RMSE        R2 
## 3.0713063 0.9907015
#c(RMSE = rmse_trn, R2=summary(model)$r.squared)

par(mfrow=c(1,1))

##################################################################################
# THE SCATTER PLOT OF EXPECTED VALUES OF TEST AND PREDICTED VALUES OF TEST ARE
# PLOTTED
##################################################################################
plot(tst_df$x, tst_df$y,main="Scatter plot of x vs y values", ylab="Expected Values of Test",xlab="X Values of Test")

plot(tst_df$x, tst_df$pred,main="Scatter plot of x vs predicted y values", ylab="Predicted Values of Test",xlab="X Values of Test")

#plot(trn_df$y, trn_df$pred)

#################################################################################
# THE SCATTER PLOT OF EXPECTED VALUES OF TEST AND PREDICTED VALUES OF TEST ARE
# PLOTTED USING GGPLOT
#################################################################################

library(ggplot2)
# Basic scatter plot
#ggplot(tst_df, aes(x=x, y=y)) + geom_point()
# Change the point size, and shape
#ggplot(tst_df, aes(x=x, y=pred)) +
#  geom_point()

ggplot(tst_df, aes(x=x, y=y)) + 
  geom_point(aes(size=2))

ggplot(tst_df, aes(x=x, y=pred)) + 
  geom_point(aes(size=2))

Inferences from the assessment of data set are

CODE BLOCK FOR DATA PREPARATION

1. The data summary of Test and Training show that Mean and Median are close which means that data has symmetrical distribution.

DATA EXPLORATION

2. Scatterplot of training and Test data show that they are linear.

3. Box plot was done to show there were no outliers

4. Density plot and Histogram was plotted to show data is normally distributed

5. Correlation between variables in x and y of Test and Training show that data is highely positively correlated.

6. Point 2 to 5 were done to show that data is good fit for Linear model

LINEAR REGRESSION MODEL AND ANALYSIS

7. Initially plot was done to show how Linear Model fits the Training data.

8. Linear Model was applied and various values extracted like Pvalue, F-Statistic etc

  1. F-statistic signifies it can be found that there is a close relationship between predictor and response variable. Here the F Statistic is 7.426e+04 this far more than 1 which means there is inter relation between predictor x and response y.

  2. Lesser the P value more important the predictor variable. The preditor variable is 2.2e-16 which is far lesser than 0.

  3. The model is fit if the R-squared(R^2) value is closer to 1. R-squared value for this model is 0.9907. So model is perfect fit.

  4. From Residuals vs Fitted graph it can be seen the red trend line is driving through zero.

  5. QQ Residuals plot is good if the points are close or on the line. We can see that it is.

USING LINEAR MODEL TO PREDICT VALUES ON TEST DATA

9. The linear Model created was used to predict on Test data. The RMSE and R Square proved that results are good.

10. Scatter plot between Expected and Predicted showed a close resemblance confirming the model worked great.

NOTE: Published the R-Mark Down in Rpubs https://rpubs.com/Nivikm/Assignment3_DM