Week11_Assignment
Abhiram
2024-04-01
library(readr)
library(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(ggplot2)mydata <- read_delim("C:\\Users\\abhir\\OneDrive\\Documents\\luddy\\H510-Stats,R\\adult.csv", delim = ",")## Rows: 32561 Columns: 15
## ── Column specification ────────────────────────────────────────────────────────
## Delimiter: ","
## chr (9): Work-Class, Education, Marital Status, Occupation, Relationship, Ra...
## dbl (6): Age, Fnl_Wgt, Education_Level, Capital_Gain, Capital_Loss, hrs_week
##
## ℹ Use `spec()` to retrieve the full column specification for this data.
## ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.mydata## # A tibble: 32,561 × 15
## Age `Work-Class` Fnl_Wgt Education Education_Level `Marital Status`
## <dbl> <chr> <dbl> <chr> <dbl> <chr>
## 1 39 " State-gov" 77516 " Bachelo… 13 " Never-married"
## 2 50 " Self-emp-not-inc" 83311 " Bachelo… 13 " Married-civ-s…
## 3 38 " Private" 215646 " HS-grad" 9 " Divorced"
## 4 53 " Private" 234721 " 11th" 7 " Married-civ-s…
## 5 28 " Private" 338409 " Bachelo… 13 " Married-civ-s…
## 6 37 " Private" 284582 " Masters" 14 " Married-civ-s…
## 7 49 " Private" 160187 " 9th" 5 " Married-spous…
## 8 52 " Self-emp-not-inc" 209642 " HS-grad" 9 " Married-civ-s…
## 9 31 " Private" 45781 " Masters" 14 " Never-married"
## 10 42 " Private" 159449 " Bachelo… 13 " Married-civ-s…
## # ℹ 32,551 more rows
## # ℹ 9 more variables: Occupation <chr>, Relationship <chr>, Race <chr>,
## # Sex <chr>, Capital_Gain <dbl>, Capital_Loss <dbl>, hrs_week <dbl>,
## # Country <chr>, Annual_pay <chr>#Generalised Linear Model
model <- lm(Age ~ Country + Education + Education_Level + hrs_week, data = mydata)
# Summarize the model
summary(model)##
## Call:
## lm(formula = Age ~ Country + Education + Education_Level + hrs_week,
## data = mydata)
##
## Residuals:
## Min 1Q Median 3Q Max
## -33.125 -10.877 -1.856 8.459 57.031
##
## Coefficients: (1 not defined because of singularities)
## Estimate Std. Error t value Pr(>|t|)
## (Intercept) 35.00851 0.73142 47.864 < 2e-16 ***
## Country Cambodia -0.63742 3.07627 -0.207 0.835851
## Country Canada 4.36756 1.31769 3.315 0.000919 ***
## Country China 2.25107 1.61991 1.390 0.164652
## Country Columbia 1.55994 1.80242 0.865 0.386788
## Country Cuba 6.67688 1.45950 4.575 4.78e-06 ***
## Country Dominican-Republic -2.37485 1.67366 -1.419 0.155921
## Country Ecuador -1.13865 2.55165 -0.446 0.655426
## Country El-Salvador -6.75376 1.40514 -4.806 1.54e-06 ***
## Country England 2.65411 1.49370 1.777 0.075600 .
## Country France -0.58558 2.50974 -0.233 0.815512
## Country Germany 1.63934 1.25249 1.309 0.190591
## Country Greece 7.21250 2.50962 2.874 0.004056 **
## Country Guatemala -9.12093 1.74736 -5.220 1.80e-07 ***
## Country Haiti 0.21652 2.06292 0.105 0.916409
## Country Holand-Netherlands -2.81263 13.19771 -0.213 0.831239
## Country Honduras -4.28535 3.69955 -1.158 0.246733
## Country Hong -6.44158 3.00062 -2.147 0.031821 *
## Country Hungary 12.33078 3.69833 3.334 0.000857 ***
## Country India -1.88852 1.43112 -1.320 0.186973
## Country Iran 0.48062 2.08510 0.231 0.817704
## Country Ireland -1.31588 2.74713 -0.479 0.631940
## Country Italy 6.47265 1.64088 3.945 8.01e-05 ***
## Country Jamaica -1.16189 1.56470 -0.743 0.457753
## Country Japan -0.16832 1.76192 -0.096 0.923894
## Country Laos -4.40630 3.15850 -1.395 0.163007
## Country Mexico -8.52890 0.78478 -10.868 < 2e-16 ***
## Country Nicaragua -3.87236 2.32799 -1.663 0.096243 .
## Country Outlying-US(Guam-USVI-etc) 1.83347 3.56686 0.514 0.607235
## Country Peru -1.63129 2.43152 -0.671 0.502293
## Country Philippines 1.45837 1.08573 1.343 0.179211
## Country Poland 4.95117 1.78832 2.769 0.005633 **
## Country Portugal -1.84184 2.24650 -0.820 0.412294
## Country Puerto-Rico 1.82482 1.35238 1.349 0.177235
## Country Scotland 2.88096 3.84602 0.749 0.453818
## Country South 0.95371 1.57269 0.606 0.544240
## Country Taiwan -6.59354 1.92869 -3.419 0.000630 ***
## Country Thailand -2.96672 3.15594 -0.940 0.347202
## Country Trinadad&Tobago 3.08370 3.07585 1.003 0.316085
## Country United-States 1.12146 0.55305 2.028 0.042593 *
## Country Vietnam -3.42011 1.70170 -2.010 0.044458 *
## Country Yugoslavia 0.73833 3.34226 0.221 0.825165
## Education 11th -4.96876 0.57877 -8.585 < 2e-16 ***
## Education 12th -5.29924 0.76744 -6.905 5.11e-12 ***
## Education 1st-4th 13.09706 1.13864 11.502 < 2e-16 ***
## Education 5th-6th 9.88705 0.88443 11.179 < 2e-16 ***
## Education 7th-8th 11.52041 0.67708 17.015 < 2e-16 ***
## Education 9th 4.49276 0.72682 6.181 6.43e-10 ***
## Education Assoc-acdm -0.51871 0.59198 -0.876 0.380911
## Education Assoc-voc 0.60311 0.55980 1.077 0.281327
## Education Bachelors 0.94040 0.46987 2.001 0.045359 *
## Education Doctorate 9.66646 0.78513 12.312 < 2e-16 ***
## Education HS-grad 1.16467 0.45124 2.581 0.009854 **
## Education Masters 6.08450 0.53879 11.293 < 2e-16 ***
## Education Preschool 9.55607 1.91367 4.994 5.96e-07 ***
## Education Prof-school 6.63046 0.70315 9.430 < 2e-16 ***
## Education Some-college -2.00389 0.45903 -4.366 1.27e-05 ***
## Education_Level NA NA NA NA
## hrs_week 0.04520 0.00604 7.483 7.43e-14 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## Residual standard error: 13.19 on 32503 degrees of freedom
## Multiple R-squared: 0.06722, Adjusted R-squared: 0.06558
## F-statistic: 41.09 on 57 and 32503 DF, p-value: < 2.2e-16This model aims to predict the variable Age using several predictor variables including Country, Education, Education_Level, and hrs_week.
Here’s a breakdown of the output:
Residuals: Shows the distribution of the differences between the observed and predicted values of the dependent variable Age. Coefficients: Provides the estimated coefficients (slopes) for each predictor variable, along with their standard errors, t-values, and p-values. The intercept represents the estimated mean age when all predictor variables are zero. Each coefficient for a predictor variable indicates the estimated change in age for a one-unit change in that predictor variable, holding other variables constant. Signif. codes: Indicates the significance level of each coefficient based on the p-values. Commonly used symbols include *** (p < 0.001), ** (p < 0.01), * (p < 0.05), . (p < 0.1), and space (p ≥ 0.1). Residual standard error: Represents the standard deviation of the residuals, indicating the average distance between the observed and predicted values of Age. Multiple R-squared: Indicates the proportion of variance in Age explained by the predictor variables. In this case, it’s approximately 6.7%, suggesting that the model explains a small portion of the variability in age. Adjusted R-squared: Similar to Multiple R-squared but adjusted for the number of predictors in the model. F-statistic: Tests the overall significance of the model by comparing the variance explained by the model to the variance not explained. p-value: Indicates the significance level of the F-statistic. A small p-value (typically < 0.05) suggests that the model is statistically significant. Overall, this output provides a summary of the linear regression model’s fit to the data, including information about the significance of individual predictors and the overall model fit.
#Tools to diagnose the model
plot(model$residuals, model$fitted.values)
# Assuming you have already fitted a linear regression model named "model"
# For demonstration purposes, let's assume "model" is the lm() object
# Predictions and residuals from the model
predictions <- model$fitted.values
residuals <- model$residuals
# Create the plot
plot(predictions, residuals,
xlab = "Fitted Values", ylab = "Residuals",
main = "Residuals vs. Fitted Values Plot",
col = "blue", pch = 16)
abline(h = 0, lty = 2) # Add a horizontal line at y = 0
residuals <- residuals(model)
# Create normal Q-Q plot
qqnorm(residuals, main = "Normal Q-Q Plot of Residuals")
qqline(residuals, col = 2)
fitted_values <- fitted(model)
std_residuals <- rstandard(model)
# Create scale-location plot
plot(fitted_values, sqrt(abs(std_residuals)),
xlab = "Fitted Values", ylab = "Square Root of Absolute Standardized Residuals",
main = "Scale-Location Plot",
pch = 16, col = "blue")
abline(h = 0, lty = 2) 
plot(model,which=5)## Warning: not plotting observations with leverage one:
## 19610
plot(model,which=4)
#Cooks Method
plot(cooks.distance(model))
Residuals vs. Fitted Values Plot:
There appears to be a pattern in the residuals vs. fitted values plot, indicating nonlinearity or heteroscedasticity in the model. This suggests that the assumption of constant variance may not hold.
Q-Q Plot of Residuals:
The Q-Q plot of residuals shows deviations from normality, indicating that the residuals may not follow a normal distribution. This violates the assumption of normally distributed residuals in the model.
Scale-Location Plot:
The scale-location plot does not show a clear pattern, but there is some evidence of heteroscedasticity, which is consistent with the findings in the residuals vs. fitted values plot.
Residuals vs. Leverage Plot:
The residuals vs. leverage plot does not indicate any influential outliers that significantly affect the model fit.
residual_deviance <- deviance(model)
df_residual <- df.residual(model)
# Calculating null deviance and degrees of freedom
null_deviance <- deviance(glm(Age ~ Country, data = mydata, family = gaussian(link = "identity")))
df_null <- nrow(data) - 1
# Calculating R-squared
r_squared <- 1 - (residual_deviance / null_deviance)
# Printing model fit statistics
cat("Residual Deviance:", residual_deviance, "\n")## Residual Deviance: 5650942cat("Degrees of Freedom (Residual):", df_residual, "\n")## Degrees of Freedom (Residual): 3250332,503 independent observations available to estimate the variability of the dependent variable after accounting for the parameters estimated in the model.
cat("Null Deviance:", null_deviance, "\n")## Null Deviance: 6010852The null deviance is 6,010,852 which indicates the deviance of the model when considering only the intercept term, without any predictor variables. It provides a baseline against which the model’s fit with predictor variables can be compared. A lower deviance in the model with predictor variables suggests that the predictors are contributing to explaining the variability in the response variable.
cat("Degrees of Freedom (Null):", df_null, "\n")## Degrees of Freedom (Null):The residual degrees of freedom, it typically refers to the number of observations minus the number of estimated parameters in the model. This value reflects the number of independent pieces of information available to estimate the variability of the data around the fitted model.
the null degrees of freedom is 9, indicating the degrees of freedom associated with the null model , while the residual degrees of freedom is 32503, indicating the degrees of freedom associated with the residuals of the fitted model.
cat("R-squared:", r_squared, "\n")## R-squared: 0.05987676R-Squared =0.05987676 suggests that approximately 5.99% of the variance in the dependent variable (Age) can be explained by the independent variable (Country) in the model. This indicates a relatively weak relationship between the independent and dependent variables.