##3a K-fold Cross Validation The dataset is randomly divided into k equal-sized folds. Each time the model fits k, one fold is held out as the validation set while the remaining folds are used for training. The held-out fold’s error is computed. This process repeats until every fold has served as the validation set one time. The final CV error estimate is the average of all the k individual error estimates.
##3b The advantages of K-fold cross validation relative to the validation set approach are that it averages over multiple splits, which gives a less biased estimate with less variability in the results. It’s also good because every observation gets used for training and testing. However the disadvantages are that K-fold cross validation is more complex and computationally expensive. Likewise, the advantages of K-fold cross validation relative to LOOCV are that K-fold is actually less computationally expensive for large datasets and reduces variance. However, K-fold does tend to increase bias.
##5
## Warning: package 'ISLR2' was built under R version 4.5.3##
## Call:
## glm(formula = default ~ income + balance, family = "binomial",
## data = Default)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -1.154e+01 4.348e-01 -26.545 < 2e-16 ***
## income 2.081e-05 4.985e-06 4.174 2.99e-05 ***
## balance 5.647e-03 2.274e-04 24.836 < 2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 2920.6 on 9999 degrees of freedom
## Residual deviance: 1579.0 on 9997 degrees of freedom
## AIC: 1585
##
## Number of Fisher Scoring iterations: 8## [1] 0.0254## [1] "Seed 2 - Validation error: 0.0238"
## [1] "Seed 3 - Validation error: 0.0264"
## [1] "Seed 4 - Validation error: 0.0256"## [1] 0.026##6a
glm_fit <- glm(default ~ income + balance, data = Default, family = "binomial")
summary(glm_fit)##
## Call:
## glm(formula = default ~ income + balance, family = "binomial",
## data = Default)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -1.154e+01 4.348e-01 -26.545 < 2e-16 ***
## income 2.081e-05 4.985e-06 4.174 2.99e-05 ***
## balance 5.647e-03 2.274e-04 24.836 < 2e-16 ***
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 2920.6 on 9999 degrees of freedom
## Residual deviance: 1579.0 on 9997 degrees of freedom
## AIC: 1585
##
## Number of Fisher Scoring iterations: 8##6b
boot.fn <- function(data, index) {
fit <- glm(default ~ income + balance, data = data, subset = index, family = "binomial")
return(coef(fit))
}
boot.fn(Default, 1:nrow(Default))## (Intercept) income balance
## -1.154047e+01 2.080898e-05 5.647103e-03##6c
set.seed(1)
boot(Default, boot.fn, R = 1000)##
## ORDINARY NONPARAMETRIC BOOTSTRAP
##
##
## Call:
## boot(data = Default, statistic = boot.fn, R = 1000)
##
##
## Bootstrap Statistics :
## original bias std. error
## t1* -1.154047e+01 -3.945460e-02 4.344722e-01
## t2* 2.080898e-05 1.680317e-07 4.866284e-06
## t3* 5.647103e-03 1.855765e-05 2.298949e-04##6d Standard errors are very similar for both glm and bootstrap. They’re essentially the same.
##9a
data(Boston)
mu_hat <- mean(Boston$medv)
mu_hat## [1] 22.53281##9b
se_mu_hat <- sd(Boston$medv) / sqrt(nrow(Boston))
se_mu_hat## [1] 0.4088611##9c
boot.fn <- function(data, index) {
return(mean(data[index]))
}
set.seed(1)
boot_mean <- boot(Boston$medv, boot.fn, R = 1000)
boot_mean##
## ORDINARY NONPARAMETRIC BOOTSTRAP
##
##
## Call:
## boot(data = Boston$medv, statistic = boot.fn, R = 1000)
##
##
## Bootstrap Statistics :
## original bias std. error
## t1* 22.53281 0.007650791 0.4106622This estimation of standard error is slightly higher.
##9d
bootstrap_se <- sd(boot_mean$t)
ci_lower <- mu_hat - 2 * bootstrap_se
ci_upper <- mu_hat + 2 * bootstrap_se
c(ci_lower, ci_upper)## [1] 21.71148 23.35413t.test(Boston$medv)##
## One Sample t-test
##
## data: Boston$medv
## t = 55.111, df = 505, p-value < 2.2e-16
## alternative hypothesis: true mean is not equal to 0
## 95 percent confidence interval:
## 21.72953 23.33608
## sample estimates:
## mean of x
## 22.53281This bootstrap confidence interval is slightly wider, but both are very similar.
##9e
mu_hat_med <- median(Boston$medv)
mu_hat_med## [1] 21.2##9f
boot.fn_median <- function(data, index) {
return(median(data[index]))
}
set.seed(1)
boot_median <- boot(Boston$medv, boot.fn_median, R = 1000)
boot_median##
## ORDINARY NONPARAMETRIC BOOTSTRAP
##
##
## Call:
## boot(data = Boston$medv, statistic = boot.fn_median, R = 1000)
##
##
## Bootstrap Statistics :
## original bias std. error
## t1* 21.2 0.02295 0.3778075The med is essentially the same here.
##9g
mu_hat_0.1 <- quantile(Boston$medv, 0.1)
mu_hat_0.1## 10%
## 12.75##9h
boot.fn_quantile <- function(data, index) {
return(quantile(data[index], 0.1))
}
set.seed(1)
boot_quantile <- boot(Boston$medv, boot.fn_quantile, R = 1000)
boot_quantile##
## ORDINARY NONPARAMETRIC BOOTSTRAP
##
##
## Call:
## boot(data = Boston$medv, statistic = boot.fn_quantile, R = 1000)
##
##
## Bootstrap Statistics :
## original bias std. error
## t1* 12.75 0.0339 0.4767526This has the largest standard error of all the bootstrap estimates, but is still relatively low.