attach(Weekly)
summary(Weekly)
## Year Lag1 Lag2 Lag3
## Min. :1990 Min. :-18.1950 Min. :-18.1950 Min. :-18.1950
## 1st Qu.:1995 1st Qu.: -1.1540 1st Qu.: -1.1540 1st Qu.: -1.1580
## Median :2000 Median : 0.2410 Median : 0.2410 Median : 0.2410
## Mean :2000 Mean : 0.1506 Mean : 0.1511 Mean : 0.1472
## 3rd Qu.:2005 3rd Qu.: 1.4050 3rd Qu.: 1.4090 3rd Qu.: 1.4090
## Max. :2010 Max. : 12.0260 Max. : 12.0260 Max. : 12.0260
## Lag4 Lag5 Volume Today
## Min. :-18.1950 Min. :-18.1950 Min. :0.08747 Min. :-18.1950
## 1st Qu.: -1.1580 1st Qu.: -1.1660 1st Qu.:0.33202 1st Qu.: -1.1540
## Median : 0.2380 Median : 0.2340 Median :1.00268 Median : 0.2410
## Mean : 0.1458 Mean : 0.1399 Mean :1.57462 Mean : 0.1499
## 3rd Qu.: 1.4090 3rd Qu.: 1.4050 3rd Qu.:2.05373 3rd Qu.: 1.4050
## Max. : 12.0260 Max. : 12.0260 Max. :9.32821 Max. : 12.0260
## Direction
## Down:484
## Up :605
##
##
##
##
corrplot(cor(Weekly[, -9]), type = "lower", diag = FALSE, method = "ellipse")
cor(Weekly[, -9])
## Year Lag1 Lag2 Lag3 Lag4
## Year 1.00000000 -0.032289274 -0.03339001 -0.03000649 -0.031127923
## Lag1 -0.03228927 1.000000000 -0.07485305 0.05863568 -0.071273876
## Lag2 -0.03339001 -0.074853051 1.00000000 -0.07572091 0.058381535
## Lag3 -0.03000649 0.058635682 -0.07572091 1.00000000 -0.075395865
## Lag4 -0.03112792 -0.071273876 0.05838153 -0.07539587 1.000000000
## Lag5 -0.03051910 -0.008183096 -0.07249948 0.06065717 -0.075675027
## Volume 0.84194162 -0.064951313 -0.08551314 -0.06928771 -0.061074617
## Today -0.03245989 -0.075031842 0.05916672 -0.07124364 -0.007825873
## Lag5 Volume Today
## Year -0.030519101 0.84194162 -0.032459894
## Lag1 -0.008183096 -0.06495131 -0.075031842
## Lag2 -0.072499482 -0.08551314 0.059166717
## Lag3 0.060657175 -0.06928771 -0.071243639
## Lag4 -0.075675027 -0.06107462 -0.007825873
## Lag5 1.000000000 -0.05851741 0.011012698
## Volume -0.058517414 1.00000000 -0.033077783
## Today 0.011012698 -0.03307778 1.000000000
Volume is strongly positively correlated with Year.
Weekly$Week <- 1:nrow(Weekly)
year_breaks <- Weekly %>%
group_by(Year) %>%
summarize(Week = min(Week))
ggplot(Weekly, aes(x = Week, y = Volume)) +
geom_line() +
geom_smooth() +
scale_x_continuous(breaks = year_breaks$Week, minor_breaks = NULL, labels = year_breaks$Year) +
labs(title = "Average Daily Shares Traded vs. Time",
x = "Time") +
theme_light()
## `geom_smooth()` using method = 'gam' and formula = 'y ~ s(x, bs = "cs")'
We notice a trend of increasing volume over time between 1990-2008.
prop.table(table(Weekly$Direction))
##
## Down Up
## 0.4444444 0.5555556
We can see the accuracy for the stocks to be up weekly is 55.56%, while the accuracy of them being down is 44.44%.
Direction as the response and the five lag variables plus
Volume as predictors. Use the summary function to print the
results. Do any of the predictors appear to be statistically
significant? If so, which ones?fit <- glm(Direction ~ Lag1+Lag2+Lag3+Lag4+Lag5+Volume,data = Weekly,family = binomial)
summary(fit)
##
## Call:
## glm(formula = Direction ~ Lag1 + Lag2 + Lag3 + Lag4 + Lag5 +
## Volume, family = binomial, data = Weekly)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.26686 0.08593 3.106 0.0019 **
## Lag1 -0.04127 0.02641 -1.563 0.1181
## Lag2 0.05844 0.02686 2.175 0.0296 *
## Lag3 -0.01606 0.02666 -0.602 0.5469
## Lag4 -0.02779 0.02646 -1.050 0.2937
## Lag5 -0.01447 0.02638 -0.549 0.5833
## Volume -0.02274 0.03690 -0.616 0.5377
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 1496.2 on 1088 degrees of freedom
## Residual deviance: 1486.4 on 1082 degrees of freedom
## AIC: 1500.4
##
## Number of Fisher Scoring iterations: 4
predictor Lag2 is significant.
contrasts(Weekly$Direction)
## Up
## Down 0
## Up 1
pred <- predict(fit, type = "response") > 0.5
(t <- table(ifelse(pred, "Up (pred)", "Down (pred)"), Weekly$Direction))
##
## Down Up
## Down (pred) 54 48
## Up (pred) 430 557
sum(diag(t)) / sum(t)
## [1] 0.5610652
Although logistic regression correctly predicts upwards movements well, it incorrectly predicts most downwards movements as up. The accuracy is 56%.
Lag2 as the only predictor.
Compute the confusion matrix and the overall fraction of correct
predictions for the held out data (that is, the data from 2009 and
2010).Weekly.train <- Weekly$Year < 2009
glm.train <- glm(Direction ~Lag2,data = Weekly[Weekly.train,],family = binomial)
summary(glm.train)
##
## Call:
## glm(formula = Direction ~ Lag2, family = binomial, data = Weekly[Weekly.train,
## ])
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) 0.20326 0.06428 3.162 0.00157 **
## Lag2 0.05810 0.02870 2.024 0.04298 *
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 1354.7 on 984 degrees of freedom
## Residual deviance: 1350.5 on 983 degrees of freedom
## AIC: 1354.5
##
## Number of Fisher Scoring iterations: 4
pred <- predict(glm.train,Weekly[!Weekly.train,], type = "response") > 0.5
(t <- table(ifelse(pred, "Up (pred)", "Down (pred)"), Weekly[!Weekly.train,]$Direction))
##
## Down Up
## Down (pred) 9 5
## Up (pred) 34 56
sum(diag(t)) / sum(t)
## [1] 0.625
Weekly.train <- Weekly$Year < 2009
lda.train <- lda(Direction ~Lag2,data = Weekly[Weekly.train,],family = binomial)
pred <- predict(lda.train,Weekly[!Weekly.train,], type = "response")$class
(t <- table(pred, Weekly[!Weekly.train,]$Direction))
##
## pred Down Up
## Down 9 5
## Up 34 56
sum(diag(t)) / sum(t)
## [1] 0.625
qda.train <- qda(Direction ~Lag2,data = Weekly[Weekly.train,],family = binomial)
pred <- predict(qda.train,Weekly[!Weekly.train,], type = "response")$class
(t <- table(pred, Weekly[!Weekly.train,]$Direction))
##
## pred Down Up
## Down 0 0
## Up 43 61
sum(diag(t)) / sum(t)
## [1] 0.5865385
fit <- knn(
Weekly[Weekly.train, "Lag2", drop = FALSE],
Weekly[!Weekly.train, "Lag2", drop = FALSE],
Weekly$Direction[Weekly.train]
)
(t <- table(fit, Weekly[!Weekly.train, ]$Direction))
##
## fit Down Up
## Down 21 30
## Up 22 31
sum(diag(t)) / sum(t)
## [1] 0.5
fit <- naiveBayes(Direction ~ Lag2, data = Smarket, subset = Weekly.train)
pred <- predict(fit, Weekly[!Weekly.train, ], type = "class")
(t <- table(pred, Weekly[!Weekly.train, ]$Direction))
##
## pred Down Up
## Down 27 29
## Up 16 32
sum(diag(t)) / sum(t)
## [1] 0.5673077
Logistic regression and LDA are the best performing.
fit <- glm(Direction ~ Lag1, data = Weekly[Weekly.train, ], family = binomial)# logistic regresion
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response") > 0.5
mean(ifelse(pred, "Up", "Down") == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5673077
fit <- glm(Direction ~ Lag3, data = Weekly[Weekly.train, ], family = binomial)#change predictor
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response") > 0.5
mean(ifelse(pred, "Up", "Down") == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5865385
fit <- glm(Direction ~Lag4, data = Weekly[Weekly.train, ], family = binomial)
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response") > 0.5
mean(ifelse(pred, "Up", "Down") == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5865385
fit <- glm(Direction ~ Lag1 + Lag2 + Lag3 + Lag4, data = Weekly[Weekly.train, ], family = binomial)
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response") > 0.5
mean(ifelse(pred, "Up", "Down") == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5865385
fit <- glm(Direction ~ Lag1 * Lag2 * Lag3 * Lag4, data = Weekly[Weekly.train, ], family = binomial)
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response") > 0.5
mean(ifelse(pred, "Up", "Down") == Weekly[!Weekly.train, ]$Direction)#check interaction
## [1] 0.5961538
fit <- lda(Direction ~ Lag1 + Lag2 + Lag3 + Lag4,data = Weekly[Weekly.train, ])
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response")$class
mean(pred == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5769231
fit <- qda(Direction ~ Lag1 + Lag2 + Lag3 + Lag4, data = Weekly[Weekly.train, ])
pred <- predict(fit, Weekly[!Weekly.train, ], type = "response")$class
mean(pred == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5192308
fit <- naiveBayes(Direction ~ Lag1 + Lag2 + Lag3 + Lag4, data = Weekly[Weekly.train, ])
pred <- predict(fit, Weekly[!Weekly.train, ], type = "class")
mean(pred == Weekly[!Weekly.train, ]$Direction)
## [1] 0.5096154
set.seed(1)
res <- sapply(1:30, function(k) {
fit <- knn(
Weekly[Weekly.train, 2:4, drop = FALSE],
Weekly[!Weekly.train, 2:4, drop = FALSE],
Weekly$Direction[Weekly.train],
k = k
)
mean(fit == Weekly[!Weekly.train, ]$Direction)
})#try k from 1 to 30
plot(1:30, res, type = "o", xlab = "k", ylab = "Fraction correct")
(k <- which.max(res))
## [1] 26
fit <- knn(
Weekly[Weekly.train, 2:4, drop = FALSE],
Weekly[!Weekly.train, 2:4, drop = FALSE],
Weekly$Direction[Weekly.train],
k = k
)
table(fit, Weekly[!Weekly.train, ]$Direction)
##
## fit Down Up
## Down 23 18
## Up 20 43
mean(fit == Weekly[!Weekly.train, ]$Direction)
## [1] 0.6346154
By using KNN, \(k = 26\) provides the best prediction(.63). Using the interaction, including lag1, lag2, lag3,lag4 , to fit logisticn regression get second(.6).
Auto
data set.mpg01, that contains a 1
if mpg contains a value above its median, and a 0 if
mpg contains a value below itsmedian. You can compute the
median using the median() function. Note you may find it
helpful to use the data.frame() function to create a single
data set containing both mpg01 and the other
Auto variables.attach(Auto)
## The following object is masked from package:lubridate:
##
## origin
## The following object is masked from package:ggplot2:
##
## mpg
med <- median(mpg)
mpg01<- ifelse (mpg>med, 1,0)
x <- cbind(Auto[, -1], data.frame(mpg01))
mpg01 and the other features. Which of
the other features seem most likely to be useful in predicting
mpg01? Scatterplots and boxplots may be useful tools to
answer this question. Describe your findings.par(mfrow = c(2, 4))
for (i in 1:7) hist(x[, i], breaks = 20, main = colnames(x)[i])
par(mfrow = c(2, 4))
for (i in 1:7) boxplot(x[, i] ~ x$mpg01, main = colnames(x)[i])
pairs(x[, 1:7])
mpg01 can differnciate most of variables.cylinders, weight, displacement, and horsepower is higly linear.
set.seed(1)
train <- sample(seq_len(nrow(x)), nrow(x) * 2/3)
summary(x[train,])
## cylinders displacement horsepower weight acceleration
## Min. :3.000 Min. : 68.0 Min. : 46.0 Min. :1649 Min. : 8.00
## 1st Qu.:4.000 1st Qu.:105.0 1st Qu.: 78.0 1st Qu.:2226 1st Qu.:14.00
## Median :4.000 Median :151.0 Median : 92.0 Median :2790 Median :15.50
## Mean :5.452 Mean :192.2 Mean :103.4 Mean :2958 Mean :15.71
## 3rd Qu.:8.000 3rd Qu.:260.0 3rd Qu.:120.0 3rd Qu.:3605 3rd Qu.:17.30
## Max. :8.000 Max. :455.0 Max. :230.0 Max. :5140 Max. :24.80
##
## year origin name mpg01
## Min. :70.00 Min. :1.000 amc gremlin : 4 Min. :0.0000
## 1st Qu.:73.00 1st Qu.:1.000 amc hornet : 4 1st Qu.:0.0000
## Median :76.00 Median :1.000 amc matador : 3 Median :1.0000
## Mean :76.11 Mean :1.529 chevrolet chevette: 3 Mean :0.5057
## 3rd Qu.:79.00 3rd Qu.:2.000 chevrolet impala : 3 3rd Qu.:1.0000
## Max. :82.00 Max. :3.000 dodge colt : 3 Max. :1.0000
## (Other) :241
mpg01 using the variables that seemed most associated with
mpg01 in (b). What is the test error of the model
obtained?fit <- lda(mpg01 ~cylinders+weight+displacement,data = x[train,])
head(fit)
## $prior
## 0 1
## 0.4942529 0.5057471
##
## $counts
## 0 1
## 129 132
##
## $means
## cylinders weight displacement
## 0 6.736434 3583.419 268.4419
## 1 4.196970 2347.720 117.6136
##
## $scaling
## LD1
## cylinders -0.3720649450
## weight -0.0007789977
## displacement -0.0035033284
##
## $lev
## [1] "0" "1"
##
## $svd
## [1] 19.67491
pred <- predict(fit, x[-train, ], type = "response")$class
mean(pred != x[-train, ]$mpg01)
## [1] 0.1068702
mpg01 using the variables that seemed most associated with
mpg01 in (b). What is the test error of the model
obtained?fit <- qda(mpg01 ~ cylinders + weight + displacement, data = x[train, ])
pred <- predict(fit, x[-train, ], type = "response")$class
mean(pred != x[-train, ]$mpg01)
## [1] 0.09923664
mpg01 using the variables that seemed most
associated with mpg01 in (b). What is the test error of the
model obtained?fit <- glm(mpg01 ~ cylinders+weight+displacement, data = x[train, ], family = binomial)
pred <- predict(fit, x[-train, ], type = "response") > 0.5
mean(pred != x[-train, ]$mpg01)
## [1] 0.1145038
mpg01 using the variables that seemed most associated with
mpg01 in (b). What is the test error of the model
obtained?fit <- naiveBayes(mpg01 ~ cylinders + weight + displacement, data = x[train, ])
pred <- predict(fit, x[-train, ], type = "class")
mean(pred != x[-train, ]$mpg01)
## [1] 0.09923664
mpg01. Use only the variables that seemed most associated
with mpg01 in (b). What test errors do you obtain? Which
value of \(K\) seems to perform the
best on this data set?res <- sapply(1:50, function(k) {
fit <- knn(x[train, c(1, 4, 2)], x[-train, c(1, 4, 2)], x$mpg01[train], k = k)
mean(fit != x[-train, ]$mpg01)
})
names(res) <- 1:50
plot(res, type = "o")
res[which.min(res)]
## 3
## 0.1068702
\(k= 4\) has the best error rate.
Boston data set, fit classification models in
order to predict whether a given census tract has a crime rate above or
below the median. Explore logistic regression, LDA, naive Bayes and KNN
models using various sub-sets of the predictors. Describe your
findings.Hint: You will have to create the response variable yourself,
using the variables that are contained in the Boston data
set.
x <- cbind(
ISLR2::Boston[, -1],
data.frame("highcrim" = Boston$crim > median(Boston$crim))
)
set.seed(1)
train <- sample(seq_len(nrow(x)), nrow(x) * 2/3)
Performing wilcox tests find the most associated variables
ord <- order(sapply(1:12, function(i) {
p <- wilcox.test(as.numeric(x[train, i]) ~ x[train, ]$highcrim)$p.value
setNames(log10(p), colnames(x)[i])
}))
ord <- names(x)[ord]
ord
## [1] "nox" "dis" "indus" "tax" "age" "rad" "zn"
## [8] "lstat" "medv" "ptratio" "rm" "chas"
Variables
nox(nitrogen oxides concentration) followed bydis(distance to employment center) appear to be most associated with high crime.
reorder columns by those most associated with highcrim (in the training data)
x <- x[, c(ord, "highcrim")]
univariate associations with highcrim (in the training
data)
x[train, ] |>
pivot_longer(!highcrim) |>
mutate(name = factor(name, levels = ord)) |>
ggplot(aes(highcrim, value)) +
geom_boxplot() +
facet_wrap(~name, scale = "free")
Fit lda, logistic regression, naive Bayes and KNN models (with k = 1..50) for a set of specific predictors and return the error rate. We fit models using increasing numbers of predictors: column 1, then columns 1 and 2 etc.
fit_models <- function(cols, k_vals = 1:50) {
dat_train <- x[train, cols, drop = FALSE]
dat_test <- x[-train, cols, drop = FALSE]
fit <- lda(x$highcrim[train] ~ ., data = dat_train)
pred <- predict(fit, dat_test, type = "response")$class
lda_err <- mean(pred != x$highcrim[-train])
fit <- glm(x$highcrim[train] ~ ., data = dat_train, family = binomial)
pred <- predict(fit, dat_test, type = "response") > 0.5
logreg_err <- mean(pred != x$highcrim[-train])
fit <- naiveBayes(x$highcrim[train] ~ ., data = dat_train)
pred <- predict(fit, dat_test, type = "class")
nb_err <- mean(pred != x$highcrim[-train])
res <- sapply(k_vals, function(k) {
fit <- knn(dat_train, dat_test, x$highcrim[train], k = k)
mean(fit != x$highcrim[-train])
})
knn_err <- min(res)
c("LDA" = lda_err, "LR" = logreg_err, "NB" = nb_err, "KNN" = knn_err)
}
res <- sapply(1:12, function(max) fit_models(1:max))
res <- as_tibble(t(res))
res$n_var <- 1:12
pivot_longer(res, cols = !n_var) |>
ggplot(aes(n_var, value, col = name)) +
geom_line() +
xlab("Number of predictors") +
ylab("Error rate")
KNN appears to perform better (if we tune \(k\)) for all numbers of predictors.
fit <- knn(
x[train, "nox", drop = FALSE],
x[-train, "nox", drop = FALSE],
x$highcrim[train],
k = 1
)
table(fit, x[-train, ]$highcrim)
##
## fit FALSE TRUE
## FALSE 78 2
## TRUE 3 86
mean(fit != x[-train, ]$highcrim) * 100
## [1] 2.95858
The best model use the method of KNN, error rate is 2.95% when \(k = 1\). The approach based on the town with the single most similar nitrogen oxide concentration (
nox). Possible explaination is nearby towns have similar crime rates, and we can obtain good predictions by predicting crime rate based on a nearby town.
Reconsider the k value, \(k = 20\)
res <- sapply(1:12, function(max) fit_models(1:max, k_vals = 20))
res <- as_tibble(t(res))
res$n_var <- 1:12
pivot_longer(res, cols = !n_var) |>
ggplot(aes(n_var, value, col = name)) +
geom_line() +
xlab("Number of predictors") +
ylab("Error rate")
KNN still performs best with a single predictor (nox),
but logistic regression with 12 predictors also performs well and has an
error rate of ~12%.
vars <- names(x)[1:12]
dat_train <- x[train, vars]
dat_test <- x[-train, vars]
fit <- glm(x$highcrim[train] ~ ., data = dat_train, family = binomial)
pred <- predict(fit, dat_test, type = "response") > 0.5
table(pred, x[-train, ]$highcrim)
##
## pred FALSE TRUE
## FALSE 70 9
## TRUE 11 79
mean(pred != x$highcrim[-train]) * 100
## [1] 11.83432
summary(fit)
##
## Call:
## glm(formula = x$highcrim[train] ~ ., family = binomial, data = dat_train)
##
## Coefficients:
## Estimate Std. Error z value Pr(>|z|)
## (Intercept) -44.525356 7.935621 -5.611 2.01e-08 ***
## nox 55.062428 10.281556 5.355 8.53e-08 ***
## dis 1.080847 0.304084 3.554 0.000379 ***
## indus -0.067493 0.058547 -1.153 0.248997
## tax -0.005336 0.003138 -1.700 0.089060 .
## age 0.020965 0.014190 1.477 0.139556
## rad 0.678196 0.192193 3.529 0.000418 ***
## zn -0.099558 0.045914 -2.168 0.030134 *
## lstat 0.134035 0.058623 2.286 0.022231 *
## medv 0.213114 0.088922 2.397 0.016547 *
## ptratio 0.294396 0.155285 1.896 0.057981 .
## rm -0.518115 0.896423 -0.578 0.563278
## chas 0.139557 0.798632 0.175 0.861280
## ---
## Signif. codes: 0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
##
## (Dispersion parameter for binomial family taken to be 1)
##
## Null deviance: 467.04 on 336 degrees of freedom
## Residual deviance: 135.80 on 324 degrees of freedom
## AIC: 161.8
##
## Number of Fisher Scoring iterations: 9