Highest traffic in I-10 Phoenix: Near Loop 101. Traffic count in that area tracked by ATR Station 100086. Need to forecast daily traffic volume

rm(list = ls())
library(dygraphs)
library(fpp3)
## ── Attaching packages ────────────────────────────────────────────── fpp3 0.5 ──
## ✔ tibble      3.2.1     ✔ tsibble     1.1.3
## ✔ dplyr       1.1.2     ✔ tsibbledata 0.4.1
## ✔ tidyr       1.3.0     ✔ feasts      0.3.1
## ✔ lubridate   1.9.2     ✔ fable       0.3.3
## ✔ ggplot2     3.4.2     ✔ fabletools  0.3.3
## ── Conflicts ───────────────────────────────────────────────── fpp3_conflicts ──
## ✖ lubridate::date()    masks base::date()
## ✖ dplyr::filter()      masks stats::filter()
## ✖ tsibble::intersect() masks base::intersect()
## ✖ tsibble::interval()  masks lubridate::interval()
## ✖ dplyr::lag()         masks stats::lag()
## ✖ tsibble::setdiff()   masks base::setdiff()
## ✖ tsibble::union()     masks base::union()
# read the data
origin = read.csv("AZ_traffic.csv")
head(origin)
df <- origin |> filter(Station_Id == 100086)
head(df)
df[['datetime']] <- as.POSIXct(df[['datetime']],
                                   format = "%Y-%m-%d %H:%M:%S")
head(df)
unique(df$Travel_Lane)
## [1] 1 2 3 4 5 6
unique(df$Travel_Dir)
## [1] 3 7
df <- df |> select(-c('Travel_Lane', 'Station_Id')) |>
  mutate(date=as.Date(datetime))
head(df)
df <- df |> select(-datetime) |>
      group_by(Travel_Dir, date) |>
      summarise(daily_volume = sum(Hour_Volume)) |> 
      ungroup()
## `summarise()` has grouped output by 'Travel_Dir'. You can override using the
## `.groups` argument.
head(df)
df <- df |>
  as_tsibble(index = date,
             key = Travel_Dir)
library(tsbox)
tsbox::ts_xts(df) |>
  dygraph() |> 
  dyRangeSelector(dateWindow = c("2018-01-01", "2019-12-30"))
## Found numeric [id] column(s): 'Travel_Dir'.
## Are you using a wide data frame? To convert, use 'ts_long()'.
## Convert column(s) to character or factor to silence this message.
## [time]: 'date' [value]: 'daily_volume'

We notice some unusually low traffic flow for some data points. After closer inspection, we observe that there are some missing dates in the time-series for which no traffic count is available. This is because ATR tracker was out of order in those days. In some cases, the tracker stopped functioning in the middle of the day. For those dates, we also observe very low traffic flow. That’s we will treat this partial records as missing values.

threshold = 61000
errors <- df |>
  filter(daily_volume < threshold)
df <- df |> anti_join(errors)
## Joining with `by = join_by(Travel_Dir, date, daily_volume)`
df <- df |> fill_gaps()
library(tsbox)
tsbox::ts_xts(df) |>
  dygraph() |> 
  dyRangeSelector(dateWindow = c("2018-01-01", "2019-12-30"))
## Found numeric [id] column(s): 'Travel_Dir'.
## Are you using a wide data frame? To convert, use 'ts_long()'.
## Convert column(s) to character or factor to silence this message.
## [time]: 'date' [value]: 'daily_volume'
missing_data <- df |> filter(is.na(daily_volume))
proxy <- origin |> filter(Station_Id == 100087)
proxy$date <- as.Date(proxy$datetime, format = "%Y-%m-%d")
proxy <- proxy |> select(-c('Travel_Lane', 'Station_Id'))
proxy <- proxy |> select(-datetime) |>
      group_by(Travel_Dir, date) |>
      summarise(daily_volume = sum(Hour_Volume)) |> 
      ungroup()
## `summarise()` has grouped output by 'Travel_Dir'. You can override using the
## `.groups` argument.
proxy <- proxy |> as_tsibble(index = date, key = Travel_Dir)
df <- df %>%
  left_join(proxy, by = c("date", "Travel_Dir"), suffix = c("", "_tsb2")) %>%
  mutate(daily_volume = coalesce(daily_volume, daily_volume_tsb2)) %>% # coalesce finds the first non-missing value
  select(-daily_volume_tsb2)

head(df)
library(tsbox)
tsbox::ts_xts(df) |>
  dygraph() |> 
  dyRangeSelector(dateWindow = c("2018-01-01", "2019-12-30"))
## Found numeric [id] column(s): 'Travel_Dir'.
## Are you using a wide data frame? To convert, use 'ts_long()'.
## Convert column(s) to character or factor to silence this message.
## [time]: 'date' [value]: 'daily_volume'
df |> has_gaps()

Data also missing in proxy

na_count <- sum(is.na(df$daily_volume))
print(na_count)
## [1] 6
df |> filter(is.na(daily_volume))
library(imputeTS)
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo
# Impute missing values using linear interpolation
df <- df %>%
  group_by(Travel_Dir) %>%
  mutate(daily_volume = na_interpolation(daily_volume, option = "linear"))
na_count <- sum(is.na(df$daily_volume))
print(na_count)
## [1] 0
df |>
  autoplot(daily_volume)
## `mutate_if()` ignored the following grouping variables:
## • Column `Travel_Dir`

df |>
  gg_season(daily_volume, period = "year", pal = c("#3333FF", "#FF3333")) +
  theme(legend.position = "left") +
  labs(y="Daily traffic count", title="Traffic volume by year") +
  facet_wrap(vars(Travel_Dir))

The annual pattern shows a uniform distribution of traffic flow throughout the year expect in March and April (due to tourism season) as well as in November and December (due to vacation period)

df |>
  gg_season(daily_volume, period = "week") +
  theme(legend.position = "none") +
  labs(y="Daily traffic count", x="Day of the week", title="Traffic Volume per week")

We notice weakly seasonality. Traffic count on weekend is a significantly lower than weekdays.

df |>
  gg_season(daily_volume, period = "month") +
  theme(legend.position = "none") +
  labs(y="Daily traffic count", x="Week of the month", title="Traffic Volume per month")

No monthly seasonality can be observed

df |>
  gg_subseries(daily_volume, period = 'week')

Same weakly pattern as seasonal plot. Less traffic on weekends. Using multiple plots, we confirm the presence of weekly seasonality

df |> filter(Travel_Dir == '3') |>
  ACF(daily_volume, lag_max = 33) |> autoplot()

The autocorrelation plot also suggests the presence of weekly seasonality

dcmp <- df |> filter(Travel_Dir == '3') |>
  model(STL(daily_volume ~ trend(window=120) + season("week", window='periodic'), robust=TRUE)) #trend(window = 90) + season('week', window=30)))
components(dcmp)|> autoplot()

components(dcmp) |>
  as_tsibble() |>
  autoplot(daily_volume, colour="gray") +
  geom_line(aes(y=trend), colour = "#D55E00") +
  labs(
    y = "Traffic Volume"
  )

From the decomposition plot, it is evident that the trend and weekly seasonality is a small part of the data. There are too much noise left in the remainder. We need some explanatory factors to explain this source of noise.

df |>
  features(daily_volume, feat_stl)
# Extract day of the week, day of the month, month, and year
df <- df %>%
  mutate(day_of_week = wday(date, label=TRUE),                              # Day of the week (e.g., Mon, Tue)
         week_of_month = week(date) - week(floor_date(date, "month")) + 1,
         month = month(date, label=TRUE))                                   # Month (e.g., Jan, Feb)
head(df)
df$week_of_month = factor(df$week_of_month, ordered = TRUE)
head(df)

We can also consider holidays like July 4th (independence day), Thanksgiving (November 23rd), etc. So, we need to add an indicator column for holidays as well.

# Define commonly observed holidays in the USA
holidays <- c("2017-01-01", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04", "2017-09-04", "2017-10-09", "2017-11-11", "2017-11-23", "2017-11-24", "2017-12-24", "2017-12-25", "2017-12-31",
                "2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04", "2018-09-03", "2018-10-08", "2018-11-11", "2018-11-22", "2018-11-23", "2018-12-24", "2018-12-25", "2018-12-31",
                "2019-01-01", "2019-01-21", "2019-02-18", "2019-05-27", "2019-07-04", "2019-09-02", "2019-10-14", "2019-11-11", "2019-11-28", "2019-11-29", "2019-12-24", "2019-12-25", "2019-12-31")
# Function to check if a date is a holiday in the USA
is_us_holiday <- function(date) {
  return(as.numeric(as.character(date) %in% holidays))
}

# Create 'is_holiday' column and set default value to FALSE
df$is_holiday <- 0

# Iterate through rows and set the 'is_holiday' indicator based on the date
for (i in 1:nrow(df)) {
  date <- df$date[i]
  year <- year(date)
  df$is_holiday[i] <- is_us_holiday(paste(year, format(date, "-%m-%d"), sep = ""))
}
head(df)
is_long_weekend <- function(date, holidays) {
  weekdays <- wday(date)
  is_holiday <- date %in% as.Date(holidays)
  is_friday_or_monday <- weekdays %in% c(2, 6)  # Monday or Friday

  ifelse(is_holiday & is_friday_or_monday, 1, 0)
}

df$is_long_weekend <- is_long_weekend(df$date, holidays)
head(df)
# Define a function to check if a date is within Thanksgiving week
is_thanksgiving_week <- function(date) {
  # Calculate Thanksgiving date (fourth Thursday in November)
  first_november <- as.Date(paste0(year(date), "-11-01"))
  offset <- 22 + (11 - as.numeric(format(first_november, "%w"))) %% 7
  thanksgiving_date <- as.Date(paste0(year(date), "-11-", offset))
  
  # Check if the date is within the week of Thanksgiving
  return(ifelse(date >= thanksgiving_date - days(3) & date <= thanksgiving_date + days(3), 1, 0))
}

# Define a function to check if a date is within Christmas week
is_christmas_week <- function(date) {
  # Assuming Christmas week is the week of December 25th
  return(ifelse(week(date) == week(as.Date(paste0(year(date), "-12-25"))), 1, 0))
}

# Define a function to check if a date is within the New Year season
is_new_year_season <- function(date) {
  # Assuming New Year season is from December 28th to January 3rd
  january_condition <- date >= (as.Date(paste0(year(date), "-12-28")) - years(1)) & date <= as.Date(paste0(year(date), "-01-03"))
  december_condition <- date >= as.Date(paste0(year(date), "-12-28")) & date <= (as.Date(paste0(year(date), "-01-03")) + years(1))
  return(ifelse(january_condition || december_condition, 1, 0))
}

# Add indicator columns for Thanksgiving week, Christmas week, and New Year season
df <- df |>
  mutate(
    is_thanksgiving_week = sapply(date, is_thanksgiving_week),
    is_christmas_week = sapply(date, is_christmas_week),
    is_new_year_season = sapply(date, is_new_year_season)
  )
df$is_long_weekend_lag2 <- lag(df$is_long_weekend, 2, default = -1)
df$is_long_weekend_lag1 <- lag(df$is_long_weekend, 1, default = -1)
df$is_long_weekend_lead2 <- lead(df$is_long_weekend, 2, default = -1)
df$is_long_weekend_lead1 <- lead(df$is_long_weekend, 1, default = -1)

df$is_holiday_lag1 <- lag(df$is_holiday, 1, default = -1)
df$is_holiday_lead1 <- lead(df$is_holiday, 1, default = -1)

df$daily_volume_lag1 <- lag(df$daily_volume, 1, default = -1)
df$daily_volume_lag2 <- lag(df$daily_volume, 2, default = -1)
df$daily_volume_lag3 <- lag(df$daily_volume, 3, default = -1)
df$daily_volume_lag4 <- lag(df$daily_volume, 4, default = -1)
df$daily_volume_lag5 <- lag(df$daily_volume, 5, default = -1)
df$daily_volume_lag6 <- lag(df$daily_volume, 6, default = -1)
df$daily_volume_lag7 <- lag(df$daily_volume, 7, default = -1)
head(df)
save(df, file="processed_data")
load("processed_data")

We want to forecast the hourly traffic for next two weeks. So, we create a test dataset with only last two weeks of data.

Train-test split

# Identify the cutoff point for the training data (all data except the last two weeks)
training_cutoff <- max(df$date) - 30

# Create training and test datasets
train <- df %>% filter(date < training_cutoff)
test <- df %>% filter(date >= training_cutoff)
head(test)

Set the baseline model

baseline_fit <- train |> filter(Travel_Dir == '3') |>
                model(Mean = MEAN(daily_volume),
                      `Naïve` = NAIVE(daily_volume),
                      `Seasonal_naïve_week` = SNAIVE(daily_volume ~ lag('week')),
                       Drift = NAIVE(daily_volume ~ drift())
                     )

# Generate forecasts for 14 quarters
baseline_fc <- baseline_fit |> forecast(h = 31)
# Plot forecasts against actual values
baseline_fc |>
  autoplot(train |> filter(Travel_Dir == '3') |> tail(30*6), level = NULL) +
  autolayer(
    filter_index(test |> filter(Travel_Dir == '3'), "2019-12-02" ~ "2019-12-31"),
    colour = "black"
  ) +
  labs(
    y = "Traffic Count",
    title = "Forecasts for daily traffic volume"
  ) +
  guides(colour = guide_legend(title = "Forecast"))
## `mutate_if()` ignored the following grouping variables:
## Plot variable not specified, automatically selected `.vars = daily_volume`
## • Column `Travel_Dir`

accuracy( baseline_fc, df|> filter(Travel_Dir == '3')) |> select(.model, MAPE, RMSE, RMSSE)

So, the ‘mean’ model performs best.

baseline_fit |> select(Mean) |> gg_tsresiduals()

# Linear Regression

dummy_vars <- model.matrix(~ week_of_month + day_of_week + month - 1, data = train)
train_reg <- cbind(train, dummy_vars) |> select(-week_of_month, -day_of_week, -month)
head(train_reg)
train_sample <- train_reg |> filter(Travel_Dir == 3)
head(train_sample)
library(leaps)
back <- regsubsets(daily_volume ~ . -daily_volume -Travel_Dir -week_of_month6 -date, data = train_sample, nvmax=40, method="backward")
coefficient.regfitBack <- coef(back, 1:40)

train.mat <- model.matrix(daily_volume ~ . -daily_volume -Travel_Dir -week_of_month6 -date, data = train_sample)
trainErrorBackward = rep(NA, 40)
for (i in 1:40) {
  coefficient = coef(back, id=i)
  prediction = train.mat[,names(coefficient)]%*%coefficient
  trainErrorBackward[i] = mean((train_sample$daily_volume - prediction)^2)
}
cat("Training errors:\n")
## Training errors:
trainErrorBackward
##  [1] 71439419 56291482 45677999 39346921 34814558 31950003 31107585 29646426
##  [9] 28164852 27126011 26474808 26070776 25653386 25433415 25292765 25110961
## [17] 24988427 24877207 24771849 24687848 24617812 24551992 24515415 24482608
## [25] 24427736 24385780 24351557 24324469 24296180 24271966 24257785 24243022
## [33] 24238082 24235126 24233397 24233339 24233302 24233275 24233251 24233244
nTrain = nrow(train_sample)
linreg.bic = nTrain*log(trainErrorBackward) + log(nTrain)*c(seq(2,41,1))
linreg.bic
##  [1] 12635.98 12476.19 12336.91 12239.32 12160.44 12107.06 12094.95 12067.92
##  [9] 12038.68 12019.00 12008.59 12004.40 11999.68 12000.22 12002.90 12004.41
## [17] 12007.55 12010.98 12014.57 12018.74 12023.31 12027.99 12033.50 12039.11
## [25] 12044.09 12049.44 12055.01 12060.78 12066.52 12072.37 12078.51 12084.63
## [33] 12091.04 12097.50 12104.00 12110.55 12117.09 12123.64 12130.19 12136.74
bestlinreg.bic = which.min(linreg.bic)
bestlinreg.bic
## [1] 13
bestcoeff.bic = coef(back,id=bestlinreg.bic)
cat("The selected variables according to BIC criteria:\n")
## The selected variables according to BIC criteria:
names(bestcoeff.bic)
##  [1] "(Intercept)"           "is_holiday"            "is_long_weekend"      
##  [4] "is_long_weekend_lead2" "daily_volume_lag1"     "day_of_week.L"        
##  [7] "day_of_week.Q"         "day_of_week.C"         "`day_of_week^4`"      
## [10] "`day_of_week^5`"       "`day_of_week^6`"       "month.C"              
## [13] "`month^4`"             "`month^7`"
linreg.aic <- nTrain*log(trainErrorBackward) + 2*c(seq(2,41,1))
linreg.aic
##  [1] 12626.88 12462.55 12318.71 12216.57 12133.15 12075.22 12058.57 12026.99
##  [9] 11993.20 11968.97 11954.01 11945.28 11936.01 11932.00 11930.13 11927.09
## [17] 11925.68 11924.56 11923.60 11923.23 11923.25 11923.38 11924.34 11925.40
## [25] 11925.84 11926.64 11927.66 11928.88 11930.07 11931.37 11932.97 11934.54
## [33] 11936.40 11938.31 11940.26 11942.26 11944.26 11946.26 11948.26 11950.26
bestlinreg.aic = which.min(linreg.aic)
bestlinreg.aic
## [1] 20
bestcoeff.aic = coef(back,id=bestlinreg.aic)
cat("The selected variables according to AIC criteria:\n")
## The selected variables according to AIC criteria:
names(bestcoeff.aic)
##  [1] "(Intercept)"           "is_holiday"            "is_long_weekend"      
##  [4] "is_thanksgiving_week"  "is_new_year_season"    "is_long_weekend_lag2" 
##  [7] "is_long_weekend_lag1"  "is_long_weekend_lead2" "daily_volume_lag1"    
## [10] "daily_volume_lag6"     "week_of_month1"        "day_of_week.L"        
## [13] "day_of_week.Q"         "day_of_week.C"         "`day_of_week^4`"      
## [16] "`day_of_week^5`"       "`day_of_week^6`"       "month.C"              
## [19] "`month^4`"             "`month^7`"             "`month^8`"
tsregFormula.bic <- as.formula(paste("daily_volume ~ trend() + ", paste0(names(bestcoeff.bic)[-1], collapse = " + ")))
tsregFormula.bic
## daily_volume ~ trend() + is_holiday + is_long_weekend + is_long_weekend_lead2 + 
##     daily_volume_lag1 + day_of_week.L + day_of_week.Q + day_of_week.C + 
##     `day_of_week^4` + `day_of_week^5` + `day_of_week^6` + month.C + 
##     `month^4` + `month^7`
fit_linreg.bic <- train_reg |>
  model(tslm = TSLM(tsregFormula.bic))
fit_linreg.bic |> filter(Travel_Dir == 3) |> report()
## Series: daily_volume 
## Model: TSLM 
## 
## Residuals:
##      Min       1Q   Median       3Q      Max 
## -28607.7  -1285.5    347.1   1967.6  24664.2 
## 
## Coefficients:
##                         Estimate Std. Error t value Pr(>|t|)    
## (Intercept)            8.490e+04  2.580e+03  32.906  < 2e-16 ***
## trend()                8.877e+00  9.709e-01   9.143  < 2e-16 ***
## is_holiday            -2.516e+04  1.853e+03 -13.575  < 2e-16 ***
## is_long_weekend        1.342e+04  2.288e+03   5.866 6.96e-09 ***
## is_long_weekend_lead2  4.800e+03  1.331e+03   3.605 0.000335 ***
## daily_volume_lag1      1.020e-01  2.729e-02   3.738 0.000201 ***
## day_of_week.L          5.912e+03  6.616e+02   8.936  < 2e-16 ***
## day_of_week.Q         -1.852e+04  5.025e+02 -36.860  < 2e-16 ***
## day_of_week.C          4.475e+03  5.081e+02   8.808  < 2e-16 ***
## `day_of_week^4`       -1.219e+04  5.730e+02 -21.269  < 2e-16 ***
## `day_of_week^5`        3.110e+03  5.462e+02   5.694 1.84e-08 ***
## `day_of_week^6`       -2.478e+03  5.044e+02  -4.912 1.13e-06 ***
## month.C                3.633e+03  6.830e+02   5.319 1.42e-07 ***
## `month^4`             -5.601e+03  6.767e+02  -8.276 6.66e-16 ***
## `month^7`             -2.925e+03  6.289e+02  -4.651 3.97e-06 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 4833 on 683 degrees of freedom
## Multiple R-squared: 0.8059,  Adjusted R-squared: 0.802
## F-statistic: 202.6 on 14 and 683 DF, p-value: < 2.22e-16
fit_linreg.bic |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = date)) +
  geom_line(aes(y = daily_volume, colour = "Data")) +
  geom_line(aes(y = .fitted, colour = "Fitted")) +
  labs(y = NULL,
    title = "Fitted traffic volume"
  ) +
  scale_colour_manual(values=c(Data="black",Fitted="#D55E00")) +
  guides(colour = guide_legend(title = NULL))

fit_linreg.bic |> filter(Travel_Dir == 3) |> gg_tsresiduals()

fit_linreg.bic |> augment() |>
  features(.innov, ljung_box, lag = 14)

As p-value is less than 0.05, we conclude that the residuals are significantly distinguishable from the white noise.

fit_linreg.bic |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = .fitted, y = .resid)) +
  geom_point() + labs(x = "Fitted", y = "Residuals")

dummy_vars <- model.matrix(~ week_of_month + day_of_week + month - 1, data = test)
test_reg <- cbind(test, dummy_vars) |> select(-week_of_month, -day_of_week, -month)
linreg.bic.fc <- fit_linreg.bic |> forecast(new_data = test_reg)
accuracy(linreg.bic.fc, df) |> select(.model, MAPE, RMSSE)
tsregFormula.aic <- as.formula(paste("daily_volume ~ trend() + ", paste0(names(bestcoeff.aic)[-1], collapse = " + ")))
tsregFormula.aic
## daily_volume ~ trend() + is_holiday + is_long_weekend + is_thanksgiving_week + 
##     is_new_year_season + is_long_weekend_lag2 + is_long_weekend_lag1 + 
##     is_long_weekend_lead2 + daily_volume_lag1 + daily_volume_lag6 + 
##     week_of_month1 + day_of_week.L + day_of_week.Q + day_of_week.C + 
##     `day_of_week^4` + `day_of_week^5` + `day_of_week^6` + month.C + 
##     `month^4` + `month^7` + `month^8`
fit_linreg.aic <- train_reg |>
  model(tslm = TSLM(tsregFormula.aic))
fit_linreg.aic |> filter(Travel_Dir == 7) |> report()
## Series: daily_volume 
## Model: TSLM 
## 
## Residuals:
##    Min     1Q Median     3Q    Max 
## -34758  -1466    275   2117  14590 
## 
## Coefficients:
##                         Estimate Std. Error t value Pr(>|t|)    
## (Intercept)            5.626e+04  4.319e+03  13.025  < 2e-16 ***
## trend()                6.424e+00  1.093e+00   5.878 6.52e-09 ***
## is_holiday            -1.789e+04  2.085e+03  -8.578  < 2e-16 ***
## is_long_weekend        1.179e+04  2.512e+03   4.692 3.28e-06 ***
## is_thanksgiving_week   2.423e+03  1.579e+03   1.535 0.125323    
## is_new_year_season    -1.343e+03  1.820e+03  -0.738 0.460833    
## is_long_weekend_lag2  -3.268e+03  1.501e+03  -2.177 0.029839 *  
## is_long_weekend_lag1  -8.419e+02  1.484e+03  -0.567 0.570722    
## is_long_weekend_lead2  4.720e+03  1.465e+03   3.222 0.001334 ** 
## daily_volume_lag1      2.160e-01  3.515e-02   6.146 1.36e-09 ***
## daily_volume_lag6      1.643e-01  3.310e-02   4.965 8.72e-07 ***
## week_of_month1        -9.516e+02  6.085e+02  -1.564 0.118302    
## day_of_week.L          1.252e+04  7.640e+02  16.383  < 2e-16 ***
## day_of_week.Q         -1.076e+04  8.437e+02 -12.755  < 2e-16 ***
## day_of_week.C         -1.813e+02  6.410e+02  -0.283 0.777361    
## `day_of_week^4`       -6.203e+03  6.069e+02 -10.222  < 2e-16 ***
## `day_of_week^5`       -1.522e+03  5.584e+02  -2.726 0.006569 ** 
## `day_of_week^6`       -1.837e+03  5.515e+02  -3.331 0.000914 ***
## month.C                2.825e+03  7.584e+02   3.725 0.000211 ***
## `month^4`             -4.285e+03  7.789e+02  -5.501 5.37e-08 ***
## `month^7`             -1.291e+03  6.892e+02  -1.874 0.061373 .  
## `month^8`              3.891e+02  6.697e+02   0.581 0.561420    
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 5176 on 676 degrees of freedom
## Multiple R-squared: 0.753,   Adjusted R-squared: 0.7454
## F-statistic: 98.15 on 21 and 676 DF, p-value: < 2.22e-16
fit_linreg.aic |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = date)) +
  geom_line(aes(y = daily_volume, colour = "Data")) +
  geom_line(aes(y = .fitted, colour = "Fitted")) +
  labs(y = NULL,
    title = "Fitted traffic volume"
  ) +
  scale_colour_manual(values=c(Data="black",Fitted="#D55E00")) +
  guides(colour = guide_legend(title = NULL))

fit_linreg.aic |> filter(Travel_Dir == 3) |> gg_tsresiduals()

fit_linreg.aic |> augment() |>
  features(.innov, ljung_box, lag = 14)

As p-value is less than 0.05 for Travel_Dir 7, we conclude that the residuals for that series are significantly distinguishable from the white noise.

fit_linreg.aic |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = .fitted, y = .resid)) +
  geom_point() + labs(x = "Fitted", y = "Residuals")

linreg.aic.fc <- fit_linreg.aic |> forecast(new_data = test_reg)
accuracy(linreg.aic.fc, df) |> select(.model, MAPE, RMSSE)

Decompositon method (Naive Bayes + Linear Regression)

fit_dcmp <- train_reg |>
  model(stlf = decomposition_model(
    STL(daily_volume ~ trend(window=120) + season("week", window=60), robust=TRUE),
    TSLM(season_adjust ~ trend() + is_holiday + is_long_weekend + is_thanksgiving_week + 
                        is_new_year_season + is_long_weekend_lag2 + is_long_weekend_lag1 + 
                        is_long_weekend_lead2 + lag(season_adjust, 1) + lag(season_adjust, 6) + 
                        week_of_month1 + month.C + `month^4` + `month^7`+`month^8`)
  ))
fit_dcmp |> filter(Travel_Dir == 7) |> gg_tsresiduals()
## Warning: Removed 7 rows containing missing values (`geom_line()`).
## Warning: Removed 7 rows containing missing values (`geom_point()`).
## Warning: Removed 7 rows containing non-finite values (`stat_bin()`).

fit_dcmp |> augment() |>
  features(.innov, ljung_box, lag = 14)
fit_dcmp |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = .fitted, y = .innov)) +
  geom_point() + labs(x = "Fitted", y = "Residuals")
## Warning: Removed 7 rows containing missing values (`geom_point()`).

fit_dcmp |> filter(Travel_Dir == 3) |> augment() |> filter(.resid > 20000)
dcmp_test_dir3 <- test_reg |> filter(Travel_Dir == '3') |>
  model(STL(daily_volume ~ trend(window=120) + season("week", window=60), robust=TRUE)) #trend(window = 90) + season('week', window=30)))

dcmp_test_dir7 <- test_reg |> filter(Travel_Dir == '7') |>
  model(STL(daily_volume ~ trend(window=120) + season("week", window=60), robust=TRUE))
test_dcmp_dir3 <- test_reg |> filter(Travel_Dir == 3) |>
  mutate(season_adjust = components(dcmp_test_dir3) |> pull(season_adjust))
test_dcmp_dir7 <- test_reg |> filter(Travel_Dir == 7) |>
  mutate(season_adjust = components(dcmp_test_dir7) |> pull(season_adjust))
test_dcmp <- bind_rows(test_dcmp_dir3, test_dcmp_dir7)
dcmp_fc <- fit_dcmp |> forecast(new_data = test_dcmp)
accuracy(dcmp_fc, df) |> select(.model, MAPE, RMSSE)

SARIMA

train |>
  features(daily_volume, unitroot_kpss)

So, the data is not stationary

train |>
  features(daily_volume, unitroot_nsdiffs)

one seasonal difference is necessary

train |> mutate(daily_volume = difference(daily_volume, 7)) |>
  features(daily_volume, unitroot_ndiffs)

No first difference is necessary

train |> filter(Travel_Dir == 3) |>
  gg_tsdisplay(difference(daily_volume, 7),
               plot_type='partial', lag=100) +
  labs(title="Seasonally differenced", y="")
## Warning: Removed 7 rows containing missing values (`geom_line()`).
## Warning: Removed 7 rows containing missing values (`geom_point()`).

D = 1, d = 0, Q = 0, P = 6, p = 39, q = 0

fit <- train |>
  model(
    auto = ARIMA(daily_volume, stepwise = FALSE, approx = FALSE)
  )
fit
glance(fit) |> arrange(AICc) |> select(Travel_Dir:BIC)
fit |> filter(Travel_Dir == 3) |> select(auto) |> gg_tsresiduals(lag=36)

augment(fit) |>
  filter(.model == "auto") |>
  features(.innov, ljung_box, lag=24, dof=4)
arima_fc <- fit |> forecast(new_data = test)
accuracy(arima_fc, df) |> select(.model, MAPE, RMSSE)

Dynamic regression

head(train_reg)
tsregFormula.aic
## daily_volume ~ trend() + is_holiday + is_long_weekend + is_thanksgiving_week + 
##     is_new_year_season + is_long_weekend_lag2 + is_long_weekend_lag1 + 
##     is_long_weekend_lead2 + daily_volume_lag1 + daily_volume_lag6 + 
##     week_of_month1 + day_of_week.L + day_of_week.Q + day_of_week.C + 
##     `day_of_week^4` + `day_of_week^5` + `day_of_week^6` + month.C + 
##     `month^4` + `month^7` + `month^8`
fit_dynreg <- train_reg |>
  model(ARIMA(daily_volume ~ trend() + is_holiday + is_long_weekend + is_thanksgiving_week + 
    is_new_year_season + is_long_weekend_lag2 + is_long_weekend_lag1 + 
    is_long_weekend_lead2 + daily_volume_lag1 + daily_volume_lag6 + 
    week_of_month1 + day_of_week.L + day_of_week.Q + day_of_week.C + 
    `day_of_week^4` + `day_of_week^5` + `day_of_week^6` + month.C + 
    `month^4` + `month^7` + `month^8` + pdq(3,0,0) + PDQ(1,0,2)))
coefficients(fit_dynreg |> filter(Travel_Dir==7))
bind_rows(
    `Regression residuals` =
        as_tibble(residuals(fit_dynreg |> filter(Travel_Dir==3), type = "regression")),
    `ARIMA residuals` =
        as_tibble(residuals(fit_dynreg |> filter(Travel_Dir==3), type = "innovation")),
    .id = "type"
  ) |>
  mutate(
    type = factor(type, levels=c(
      "Regression residuals", "ARIMA residuals"))
  ) |>
  ggplot(aes(x = date, y = .resid)) +
  geom_line() +
  facet_grid(vars(type))

fit_dynreg |> filter(Travel_Dir==3) |> gg_tsresiduals()

augment(fit_dynreg) |>
  features(.innov, ljung_box, lag = 14)
dynreg_fc <- fit_dynreg |> forecast(new_data = test_reg)
accuracy(dynreg_fc, df) |> select(.model, MAPE, RMSSE)

Decomposition (Naive Bayes + Dynamic Regression)

fit_arima_dcmp <- train_reg |>
  model(stlf = decomposition_model(
    STL(daily_volume ~ trend(window=120) + season("week", window=60), robust=TRUE),
    ARIMA(season_adjust ~ trend() + is_holiday + is_long_weekend + is_thanksgiving_week + 
                        is_new_year_season + is_long_weekend_lag2 + is_long_weekend_lag1 + 
                        is_long_weekend_lead2 + lag(season_adjust, 1) + lag(season_adjust, 2) + lag(season_adjust, 3) +
                        month.C + `month^4` + `month^7` + `month^8` + `month^9`, stepwise = FALSE, approximation = FALSE)
  ))
fit_arima_dcmp |> filter(Travel_Dir == 3) |> gg_tsresiduals()
## Warning: Removed 7 rows containing missing values (`geom_line()`).
## Warning: Removed 7 rows containing missing values (`geom_point()`).
## Warning: Removed 7 rows containing non-finite values (`stat_bin()`).

fit_arima_dcmp |> filter(Travel_Dir == 3) |> report()
## Series: daily_volume 
## Model: STL decomposition model 
## Combination: season_adjust + season_week
## 
## ========================================
## 
## Series: season_adjust 
## Model: LM w/ ARIMA(0,0,0)(1,0,1)[7] errors 
## 
## Coefficients:
##         sar1     sma1  trend()  is_holiday  is_long_weekend
##       0.8815  -0.8412   7.7987  -26919.639        15006.137
## s.e.  0.1221   0.1398   1.2321    1857.119         2185.928
##       is_thanksgiving_week  is_new_year_season  is_long_weekend_lag2
##                   2539.881             35.4794               44.2939
## s.e.              1397.337           1822.2034             1337.9996
##       is_long_weekend_lag1  is_long_weekend_lead2  lag(season_adjust, 1)
##                   4689.844               4490.225                 0.1899
## s.e.              1328.271               1241.775                 0.0332
##       lag(season_adjust, 2)  lag(season_adjust, 3)    month.C  `month^4`
##                      0.0265                -0.0079  3277.3572  -4606.580
## s.e.                 0.0341                 0.0325   735.3434    750.049
##        `month^7`  `month^8`  `month^9`  intercept
##       -2481.7941  1297.5434  -852.2998  74773.139
## s.e.    628.4171   602.5244   593.6297   4885.812
## 
## sigma^2 estimated as 21622101:  log likelihood=-6847.16
## AIC=13734.32   AICc=13735.56   BIC=13825.29
## 
## Series: season_week 
## Model: SNAIVE 
## 
## sigma^2: 534.5482
augment(fit_arima_dcmp) |>
  features(.innov, ljung_box, lag = 14)
dcmp_arima_fc <- fit_arima_dcmp |> forecast(new_data = test_dcmp)
accuracy(dcmp_arima_fc, df) |> select(.model, MAPE, RMSSE)
fit_arima_dcmp |> filter(Travel_Dir == 3) |> augment() |>
  ggplot(aes(x = .fitted, y = .innov)) +
  geom_point() + labs(x = "Fitted", y = "Residuals")
## Warning: Removed 7 rows containing missing values (`geom_point()`).

forecast(fit_arima_dcmp, new_data = test_dcmp) |>
  filter(Travel_Dir == 3) |>
  autoplot(tail(train |> filter(Travel_Dir == 3), 90)) +
  autolayer(
    test |> filter(Travel_Dir == 3),
    colour = "black"
  ) +
  labs(title = "Traffic Volume Forecast",
       y="Traffic count")
## `mutate_if()` ignored the following grouping variables:
## Plot variable not specified, automatically selected `.vars = daily_volume`
## • Column `Travel_Dir`