Licença

This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/ or send a letter to Creative Commons, PO Box 1866, Mountain View, CA 94042, USA.

License: CC BY-SA 4.0

Citação

Sugestão de citação: FIGUEIREDO, Adriano Marcos Rodrigues. Exemplo Séries Temporais: modeltime - AirPassengers. Campo Grande-MS, Brasil: RStudio/Rpubs, 2023. Disponível em https://rpubs.com/amrofi/modeltime_airpassengers.

Introdução

Os dados vem do pacote datasets, AirPassengers, seguindo o exemplo da vignette do pacote modeltime (Dancho, 2023).

0.1 Carregar pacotes

library(xgboost)
library(tidymodels)
library(modeltime)
library(modeltime.ensemble)
library(tidyverse)
library(lubridate)
library(timetk)
library(zoo)
# This toggles plots from plotly (interactive) to ggplot (static)
interactive <- FALSE

1 Passo 1: dados

# Load the data
data(AirPassengers)  # ts format
#
df<-data.frame(date = as.Date(as.yearmon(time(AirPassengers))),
               value=as.matrix(AirPassengers))

1.1 Visualização de dados

# visualize
df %>%
  plot_time_series(date, value, .interactive = interactive)

2 Passo 2: separar amostras treino x teste

# Split the data into train and test sets
# Split Data 80/20
splits <- initial_time_split(df, prop = 0.8)  # 80:20
#

3 Passo 3: modelagem

3.1 Model 1: Auto ARIMA (Modeltime)

## Model 1: auto_arima
model_fit_arima_no_boost <- arima_reg() %>%
  set_engine(engine = "auto_arima") %>%
  fit(value ~ date, data = training(splits))
# frequency = 12 observations per 1 year

3.2 Model 2: Boosted Auto ARIMA (Modeltime)

## Model 2: arima_boost
model_fit_arima_boosted <- arima_boost(
  min_n = 2,
  learn_rate = 0.015
) %>%
  set_engine(engine = "auto_arima_xgboost") %>%
  fit(value ~ date + as.numeric(date) + factor(month(date, label = TRUE), ordered = F),
      data = training(splits))
# frequency = 12 observations per 1 year

3.3 Model 3: Exponential Smoothing - ETS (Modeltime)

## Model 3: ets
model_fit_ets <- exp_smoothing() %>%
  set_engine(engine = "ets") %>%
  fit(value ~ date, data = training(splits))
#> frequency = 12 observations per 1 year

3.4 Model 4: Prophet (Modeltime)

## Model 4: prophet 
model_fit_prophet <- prophet_reg() %>%
  set_engine(engine = "prophet") %>%
  fit(value ~ date, data = training(splits))
#> Disabling weekly seasonality. Run prophet with weekly.seasonality=TRUE to override this.
#> Disabling daily seasonality. Run prophet with daily.seasonality=TRUE to override this.

3.5 Model 5: Linear Regression (Parsnip)

# Model 5: lm 
model_fit_lm <- linear_reg() %>%
  set_engine("lm") %>%
  fit(value ~ as.numeric(date) + factor(month(date, label = TRUE), ordered = FALSE),
      data = training(splits))

3.6 Model 6: MARS (Workflow)

# Multivariate Adaptive Regression Spline model 
# Model 6: earth ----
model_spec_mars <- mars(mode = "regression") %>%
  set_engine("earth") 

recipe_spec <- recipe(value ~ date, data = training(splits)) %>%
  step_date(date, features = "month", ordinal = FALSE) %>%
  step_mutate(date_num = as.numeric(date)) %>%
  step_normalize(date_num) %>%
  step_rm(date)
library(earth)
wflw_fit_mars <- workflow() %>%
  add_recipe(recipe_spec) %>%
  add_model(model_spec_mars) %>%
  fit(training(splits))

3.7 Model 7: Theta method

# Model 7 - Theta method
model_fit_theta <- exp_smoothing() %>%
  set_engine(engine = "theta") %>%
  fit(value ~ date, data = training(splits))
# frequency = 12 observations per 1 year

4 Passo 4: Add fitted models to a Model Table

models_tbl <- modeltime_table(
  model_fit_arima_no_boost,
  model_fit_arima_boosted,
  model_fit_ets,
  model_fit_prophet,
  model_fit_lm,
  wflw_fit_mars,
  model_fit_theta
)

models_tbl

5 Passo 5: Calibrate the model to a testing set

calibration_tbl <- models_tbl %>%
  modeltime_calibrate(new_data = testing(splits))

calibration_tbl

6 Passo 6: Testing Set Forecast & Accuracy Evaluation

calibration_tbl %>%
  modeltime_forecast(
    new_data    = testing(splits),
    actual_data = df
  ) %>%
  plot_modeltime_forecast(
    .legend_max_width = 25, # For mobile screens
    .interactive      = interactive
  )

6.1 Accuracy Metrics

calibration_tbl %>%
  modeltime_accuracy() %>%
  table_modeltime_accuracy(
    .interactive = FALSE
  )
Accuracy Table
.model_id .model_desc .type mae mape mase smape rmse rsq
1 ARIMA(1,1,0)(0,1,0)[12] Test 28.55 6.12 0.62 6.35 35.08 0.93
2 ARIMA(1,1,0)(0,1,0)[12] W/ XGBOOST ERRORS Test 28.14 6.04 0.61 6.26 34.65 0.93
3 ETS(M,AD,M) Test 29.98 6.53 0.65 6.68 35.81 0.89
4 PROPHET Test 33.84 7.70 0.74 7.62 41.31 0.91
5 LM Test 34.31 7.08 0.75 7.39 47.55 0.95
6 EARTH Test 57.51 11.88 1.26 12.92 73.02 0.86
7 THETA METHOD Test 90.63 22.70 1.98 19.86 101.55 0.20

6.2 Refit to Full Dataset & Forecast Forward

refit_tbl <- calibration_tbl %>%
  modeltime_refit(data = df)

refit_tbl %>%
  modeltime_forecast(h = "3 years", actual_data = df) %>%
  plot_modeltime_forecast(
    .legend_max_width = 25, # For mobile screens
    .interactive      = interactive
  )

7 Model 8: Ensemble

Faremos a rotina de combinação de forecasts para os modelos auto ARIMA, Prophet, Elastic Net e ETS.

# https://business-science.github.io/modeltime.ensemble/articles/getting-started-with-modeltime-ensemble.html
# Recipe
recipe_spec <- recipe(value ~ date, training(splits)) %>%
    step_timeseries_signature(date) %>%
    step_rm(matches("(.iso$)|(.xts$)")) %>%
    step_normalize(matches("(index.num$)|(_year$)")) %>%
    step_dummy(all_nominal()) %>%
    step_fourier(date, K = 1, period = 12)

recipe_spec %>% prep() %>% juice()
# Model 1 auto ARIMA
model_spec_arima <- arima_reg() %>%
    set_engine("auto_arima")

wflw_fit_arima <- workflow() %>%
    add_model(model_spec_arima) %>%
    add_recipe(recipe_spec %>% step_rm(all_predictors(), -date)) %>%
    fit(training(splits))

# Model 2 - Prophet
model_spec_prophet <- prophet_reg() %>%
    set_engine("prophet")

wflw_fit_prophet <- workflow() %>%
    add_model(model_spec_prophet) %>%
    add_recipe(recipe_spec %>% step_rm(all_predictors(), -date)) %>%
    fit(training(splits))

# Model 3 - Elastic Net
model_spec_glmnet <- linear_reg(
    mixture = 0.9,
    penalty = 4.36e-6 ) %>%
    set_engine("glmnet")

wflw_fit_glmnet <- workflow() %>%
    add_model(model_spec_glmnet) %>%
    add_recipe(recipe_spec %>% step_rm(date)) %>%
    fit(training(splits))

# Model 4 - ETS
model_spec_ets <- exp_smoothing() %>%
    set_engine(engine = "ets")
  
wflw_fit_ets <- workflow() %>%
    add_model(model_spec_ets) %>%
    add_recipe(recipe_spec %>%
step_rm(all_predictors(), -date)) %>%
    fit(training(splits))

7.1 Step 1 - Create a Modeltime Table

airpass_models <- modeltime_table(
    wflw_fit_arima,
    wflw_fit_prophet,
    wflw_fit_glmnet,
    wflw_fit_ets
)

airpass_models

7.2 Step 2 - Make an Ensemble

ensemble_fit <- airpass_models %>%
    ensemble_average(type = "mean")

ensemble_fit
## ── Modeltime Ensemble ───────────────────────────────────────────
## Ensemble of 4 Models (MEAN)
## 
## # Modeltime Table
## # A tibble: 4 × 3
##   .model_id .model     .model_desc            
##       <int> <list>     <chr>                  
## 1         1 <workflow> ARIMA(1,1,0)(0,1,0)[12]
## 2         2 <workflow> PROPHET                
## 3         3 <workflow> GLMNET                 
## 4         4 <workflow> ETS(M,AD,M)

7.3 Step 3 - Forecast! (the Test Data)

# Calibration
calibration_tbl <- modeltime_table(
    ensemble_fit
) %>%
    modeltime_calibrate(testing(splits))

# Forecast vs Test Set
calibration_tbl %>%
    modeltime_forecast(
        new_data    = testing(splits),
        actual_data = df
    ) %>%
    plot_modeltime_forecast(.interactive = T)

7.4 Step 4 - Refit on Full Data & Forecast Future

calibration_tbl %>%
  modeltime_accuracy() %>%
  table_modeltime_accuracy(
    .interactive = T
  )
refit_tbl <- calibration_tbl %>%
    modeltime_refit(df)

refit_tbl %>%
    modeltime_forecast(
        h = "2 years",
        actual_data = df
    ) %>%
    plot_modeltime_forecast(.interactive = T)

Referências

Dancho M (2023). modeltime: The Tidymodels Extension for Time Series Modeling. R package version 1.2.6, https://CRAN.R-project.org/package=modeltime.

LS0tDQp0aXRsZTogIkV4ZW1wbG8gU8OpcmllcyBUZW1wb3JhaXM6IG1vZGVsdGltZSAtIEFpclBhc3NlbmdlcnMiDQphdXRob3I6ICJBZHJpYW5vIE1hcmNvcyBSb2RyaWd1ZXMgRmlndWVpcmVkbywgKmUtbWFpbDogYWRyaWFuby5maWd1ZWlyZWRvQHVmbXMuYnIqIg0KZGF0ZTogImByIGZvcm1hdChTeXMuRGF0ZSgpLCAnJWQgJUIgJVknKWAiDQpvdXRwdXQ6DQogIGh0bWxfZG9jdW1lbnQ6DQogICAgY29kZV9kb3dubG9hZDogeWVzDQogICAgdGhlbWU6IGRlZmF1bHQNCiAgICBudW1iZXJfc2VjdGlvbnM6IHllcw0KICAgIHRvYzogeWVzDQogICAgdG9jX2Zsb2F0OiBubw0KICAgIGRmX3ByaW50OiBwYWdlZA0KICAgIGZpZ19jYXB0aW9uOiB5ZXMNCi0tLQ0KDQpgYGB7ciBzZXR1cCwgaW5jbHVkZT1GQUxTRX0NCmtuaXRyOjpvcHRzX2NodW5rJHNldChlY2hvID0gVFJVRSkNCmBgYA0KDQojIExpY2Vuw6dhIHsjTGljZW7Dp2EgLnVubnVtYmVyZWR9DQoNClRoaXMgd29yayBpcyBsaWNlbnNlZCB1bmRlciB0aGUgQ3JlYXRpdmUgQ29tbW9ucyBBdHRyaWJ1dGlvbi1TaGFyZUFsaWtlIDQuMCBJbnRlcm5hdGlvbmFsIExpY2Vuc2UuIFRvIHZpZXcgYSBjb3B5IG9mIHRoaXMgbGljZW5zZSwgdmlzaXQgPGh0dHA6Ly9jcmVhdGl2ZWNvbW1vbnMub3JnL2xpY2Vuc2VzL2J5LXNhLzQuMC8+IG9yIHNlbmQgYSBsZXR0ZXIgdG8gQ3JlYXRpdmUgQ29tbW9ucywgUE8gQm94IDE4NjYsIE1vdW50YWluIFZpZXcsIENBIDk0MDQyLCBVU0EuDQoNCiFbTGljZW5zZTogQ0MgQlktU0EgNC4wXShodHRwczovL21pcnJvcnMuY3JlYXRpdmVjb21tb25zLm9yZy9wcmVzc2tpdC9idXR0b25zLzg4eDMxL3BuZy9ieS1zYS5wbmcpe3dpZHRoPSIyNSUifQ0KDQojIENpdGHDp8OjbyB7I0NpdGHDp8OjbyAudW5udW1iZXJlZH0NCg0KU3VnZXN0w6NvIGRlIGNpdGHDp8OjbzogRklHVUVJUkVETywgQWRyaWFubyBNYXJjb3MgUm9kcmlndWVzLiBFeGVtcGxvIFPDqXJpZXMgVGVtcG9yYWlzOiBtb2RlbHRpbWUgLSBBaXJQYXNzZW5nZXJzLiBDYW1wbyBHcmFuZGUtTVMsIEJyYXNpbDogUlN0dWRpby9ScHVicywgMjAyMy4gRGlzcG9uw612ZWwgZW0gPGh0dHBzOi8vcnB1YnMuY29tL2Ftcm9maS9tb2RlbHRpbWVfYWlycGFzc2VuZ2Vycz4uDQoNCiMgSW50cm9kdcOnw6NvIHsjSW50cm9kdcOnw6NvIC51bm51bWJlcmVkfQ0KDQpPcyBkYWRvcyB2ZW0gZG8gcGFjb3RlIGRhdGFzZXRzLCBBaXJQYXNzZW5nZXJzLCBzZWd1aW5kbyBvIGV4ZW1wbG8gZGEgdmlnbmV0dGUgZG8gcGFjb3RlIG1vZGVsdGltZSAoRGFuY2hvLCAyMDIzKS4NCg0KIyMgQ2FycmVnYXIgcGFjb3Rlcw0KDQpgYGB7ciBwYWNvdGVzLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KbGlicmFyeSh4Z2Jvb3N0KQ0KbGlicmFyeSh0aWR5bW9kZWxzKQ0KbGlicmFyeShtb2RlbHRpbWUpDQpsaWJyYXJ5KG1vZGVsdGltZS5lbnNlbWJsZSkNCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShsdWJyaWRhdGUpDQpsaWJyYXJ5KHRpbWV0aykNCmxpYnJhcnkoem9vKQ0KIyBUaGlzIHRvZ2dsZXMgcGxvdHMgZnJvbSBwbG90bHkgKGludGVyYWN0aXZlKSB0byBnZ3Bsb3QgKHN0YXRpYykNCmludGVyYWN0aXZlIDwtIEZBTFNFDQpgYGANCg0KIyBQYXNzbyAxOiBkYWRvcw0KDQpgYGB7ciBwYXNzbzEsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQojIExvYWQgdGhlIGRhdGENCmRhdGEoQWlyUGFzc2VuZ2VycykgICMgdHMgZm9ybWF0DQojDQpkZjwtZGF0YS5mcmFtZShkYXRlID0gYXMuRGF0ZShhcy55ZWFybW9uKHRpbWUoQWlyUGFzc2VuZ2VycykpKSwNCiAgICAgICAgICAgICAgIHZhbHVlPWFzLm1hdHJpeChBaXJQYXNzZW5nZXJzKSkNCmBgYA0KDQojIyBWaXN1YWxpemHDp8OjbyBkZSBkYWRvcw0KDQpgYGB7ciB2aXN1YWxkYWRvcywgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCg0KIyB2aXN1YWxpemUNCmRmICU+JQ0KICBwbG90X3RpbWVfc2VyaWVzKGRhdGUsIHZhbHVlLCAuaW50ZXJhY3RpdmUgPSBpbnRlcmFjdGl2ZSkNCmBgYA0KDQojIFBhc3NvIDI6IHNlcGFyYXIgYW1vc3RyYXMgdHJlaW5vIHggdGVzdGUNCg0KYGBge3IgcGFzc28yLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KDQojIFNwbGl0IHRoZSBkYXRhIGludG8gdHJhaW4gYW5kIHRlc3Qgc2V0cw0KIyBTcGxpdCBEYXRhIDgwLzIwDQpzcGxpdHMgPC0gaW5pdGlhbF90aW1lX3NwbGl0KGRmLCBwcm9wID0gMC44KSAgIyA4MDoyMA0KIw0KYGBgDQoNCiMgUGFzc28gMzogbW9kZWxhZ2VtDQoNCiMjIE1vZGVsIDE6IEF1dG8gQVJJTUEgKE1vZGVsdGltZSkgDQoNCmBgYHtyIG1vZDEsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQoNCiMjIE1vZGVsIDE6IGF1dG9fYXJpbWENCm1vZGVsX2ZpdF9hcmltYV9ub19ib29zdCA8LSBhcmltYV9yZWcoKSAlPiUNCiAgc2V0X2VuZ2luZShlbmdpbmUgPSAiYXV0b19hcmltYSIpICU+JQ0KICBmaXQodmFsdWUgfiBkYXRlLCBkYXRhID0gdHJhaW5pbmcoc3BsaXRzKSkNCiMgZnJlcXVlbmN5ID0gMTIgb2JzZXJ2YXRpb25zIHBlciAxIHllYXINCg0KYGBgDQoNCiMjIE1vZGVsIDI6IEJvb3N0ZWQgQXV0byBBUklNQSAoTW9kZWx0aW1lKQ0KDQpgYGB7ciBtb2QyLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyMgTW9kZWwgMjogYXJpbWFfYm9vc3QNCm1vZGVsX2ZpdF9hcmltYV9ib29zdGVkIDwtIGFyaW1hX2Jvb3N0KA0KICBtaW5fbiA9IDIsDQogIGxlYXJuX3JhdGUgPSAwLjAxNQ0KKSAlPiUNCiAgc2V0X2VuZ2luZShlbmdpbmUgPSAiYXV0b19hcmltYV94Z2Jvb3N0IikgJT4lDQogIGZpdCh2YWx1ZSB+IGRhdGUgKyBhcy5udW1lcmljKGRhdGUpICsgZmFjdG9yKG1vbnRoKGRhdGUsIGxhYmVsID0gVFJVRSksIG9yZGVyZWQgPSBGKSwNCiAgICAgIGRhdGEgPSB0cmFpbmluZyhzcGxpdHMpKQ0KIyBmcmVxdWVuY3kgPSAxMiBvYnNlcnZhdGlvbnMgcGVyIDEgeWVhcg0KYGBgDQoNCiMjIE1vZGVsIDM6IEV4cG9uZW50aWFsIFNtb290aGluZyAtIEVUUyAoTW9kZWx0aW1lKQ0KDQpgYGB7ciBtb2QzLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyMgTW9kZWwgMzogZXRzDQptb2RlbF9maXRfZXRzIDwtIGV4cF9zbW9vdGhpbmcoKSAlPiUNCiAgc2V0X2VuZ2luZShlbmdpbmUgPSAiZXRzIikgJT4lDQogIGZpdCh2YWx1ZSB+IGRhdGUsIGRhdGEgPSB0cmFpbmluZyhzcGxpdHMpKQ0KIz4gZnJlcXVlbmN5ID0gMTIgb2JzZXJ2YXRpb25zIHBlciAxIHllYXINCmBgYA0KDQojIyBNb2RlbCA0OiBQcm9waGV0IChNb2RlbHRpbWUpIA0KDQpgYGB7ciBtb2Q0LCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KDQojIyBNb2RlbCA0OiBwcm9waGV0IA0KbW9kZWxfZml0X3Byb3BoZXQgPC0gcHJvcGhldF9yZWcoKSAlPiUNCiAgc2V0X2VuZ2luZShlbmdpbmUgPSAicHJvcGhldCIpICU+JQ0KICBmaXQodmFsdWUgfiBkYXRlLCBkYXRhID0gdHJhaW5pbmcoc3BsaXRzKSkNCiM+IERpc2FibGluZyB3ZWVrbHkgc2Vhc29uYWxpdHkuIFJ1biBwcm9waGV0IHdpdGggd2Vla2x5LnNlYXNvbmFsaXR5PVRSVUUgdG8gb3ZlcnJpZGUgdGhpcy4NCiM+IERpc2FibGluZyBkYWlseSBzZWFzb25hbGl0eS4gUnVuIHByb3BoZXQgd2l0aCBkYWlseS5zZWFzb25hbGl0eT1UUlVFIHRvIG92ZXJyaWRlIHRoaXMuDQpgYGANCg0KDQoNCiMjIE1vZGVsIDU6IExpbmVhciBSZWdyZXNzaW9uIChQYXJzbmlwKQ0KDQpgYGB7ciBtb2Q1LCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyBNb2RlbCA1OiBsbSANCm1vZGVsX2ZpdF9sbSA8LSBsaW5lYXJfcmVnKCkgJT4lDQogIHNldF9lbmdpbmUoImxtIikgJT4lDQogIGZpdCh2YWx1ZSB+IGFzLm51bWVyaWMoZGF0ZSkgKyBmYWN0b3IobW9udGgoZGF0ZSwgbGFiZWwgPSBUUlVFKSwgb3JkZXJlZCA9IEZBTFNFKSwNCiAgICAgIGRhdGEgPSB0cmFpbmluZyhzcGxpdHMpKQ0KYGBgDQoNCg0KIyMgTW9kZWwgNjogTUFSUyAoV29ya2Zsb3cpDQoNCmBgYHtyIG1vZDYsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQojIE11bHRpdmFyaWF0ZSBBZGFwdGl2ZSBSZWdyZXNzaW9uIFNwbGluZSBtb2RlbCANCiMgTW9kZWwgNjogZWFydGggLS0tLQ0KbW9kZWxfc3BlY19tYXJzIDwtIG1hcnMobW9kZSA9ICJyZWdyZXNzaW9uIikgJT4lDQogIHNldF9lbmdpbmUoImVhcnRoIikgDQoNCnJlY2lwZV9zcGVjIDwtIHJlY2lwZSh2YWx1ZSB+IGRhdGUsIGRhdGEgPSB0cmFpbmluZyhzcGxpdHMpKSAlPiUNCiAgc3RlcF9kYXRlKGRhdGUsIGZlYXR1cmVzID0gIm1vbnRoIiwgb3JkaW5hbCA9IEZBTFNFKSAlPiUNCiAgc3RlcF9tdXRhdGUoZGF0ZV9udW0gPSBhcy5udW1lcmljKGRhdGUpKSAlPiUNCiAgc3RlcF9ub3JtYWxpemUoZGF0ZV9udW0pICU+JQ0KICBzdGVwX3JtKGRhdGUpDQpsaWJyYXJ5KGVhcnRoKQ0Kd2Zsd19maXRfbWFycyA8LSB3b3JrZmxvdygpICU+JQ0KICBhZGRfcmVjaXBlKHJlY2lwZV9zcGVjKSAlPiUNCiAgYWRkX21vZGVsKG1vZGVsX3NwZWNfbWFycykgJT4lDQogIGZpdCh0cmFpbmluZyhzcGxpdHMpKQ0KYGBgDQoNCiMjIE1vZGVsIDc6IFRoZXRhIG1ldGhvZA0KDQpgYGB7ciBtb2Q3LCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KIyBNb2RlbCA3IC0gVGhldGEgbWV0aG9kDQptb2RlbF9maXRfdGhldGEgPC0gZXhwX3Ntb290aGluZygpICU+JQ0KICBzZXRfZW5naW5lKGVuZ2luZSA9ICJ0aGV0YSIpICU+JQ0KICBmaXQodmFsdWUgfiBkYXRlLCBkYXRhID0gdHJhaW5pbmcoc3BsaXRzKSkNCiMgZnJlcXVlbmN5ID0gMTIgb2JzZXJ2YXRpb25zIHBlciAxIHllYXINCmBgYA0KDQoNCg0KIyBQYXNzbyA0OiBBZGQgZml0dGVkIG1vZGVscyB0byBhIE1vZGVsIFRhYmxlDQoNCmBgYHtyIGZpdHRlZF9tb2RlbHMsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQptb2RlbHNfdGJsIDwtIG1vZGVsdGltZV90YWJsZSgNCiAgbW9kZWxfZml0X2FyaW1hX25vX2Jvb3N0LA0KICBtb2RlbF9maXRfYXJpbWFfYm9vc3RlZCwNCiAgbW9kZWxfZml0X2V0cywNCiAgbW9kZWxfZml0X3Byb3BoZXQsDQogIG1vZGVsX2ZpdF9sbSwNCiAgd2Zsd19maXRfbWFycywNCiAgbW9kZWxfZml0X3RoZXRhDQopDQoNCm1vZGVsc190YmwNCmBgYA0KDQojIFBhc3NvIDU6IENhbGlicmF0ZSB0aGUgbW9kZWwgdG8gYSB0ZXN0aW5nIHNldA0KDQpgYGB7ciBjYWxpYnJhdGUsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQpjYWxpYnJhdGlvbl90YmwgPC0gbW9kZWxzX3RibCAlPiUNCiAgbW9kZWx0aW1lX2NhbGlicmF0ZShuZXdfZGF0YSA9IHRlc3Rpbmcoc3BsaXRzKSkNCg0KY2FsaWJyYXRpb25fdGJsDQpgYGANCg0KIyBQYXNzbyA2OiBUZXN0aW5nIFNldCBGb3JlY2FzdCAmIEFjY3VyYWN5IEV2YWx1YXRpb24NCg0KYGBge3IgZm9yZWNhc3QsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQpjYWxpYnJhdGlvbl90YmwgJT4lDQogIG1vZGVsdGltZV9mb3JlY2FzdCgNCiAgICBuZXdfZGF0YSAgICA9IHRlc3Rpbmcoc3BsaXRzKSwNCiAgICBhY3R1YWxfZGF0YSA9IGRmDQogICkgJT4lDQogIHBsb3RfbW9kZWx0aW1lX2ZvcmVjYXN0KA0KICAgIC5sZWdlbmRfbWF4X3dpZHRoID0gMjUsICMgRm9yIG1vYmlsZSBzY3JlZW5zDQogICAgLmludGVyYWN0aXZlICAgICAgPSBpbnRlcmFjdGl2ZQ0KICApDQpgYGANCg0KIyMgQWNjdXJhY3kgTWV0cmljcw0KDQpgYGB7ciBhY2N1cmFjeSwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCmNhbGlicmF0aW9uX3RibCAlPiUNCiAgbW9kZWx0aW1lX2FjY3VyYWN5KCkgJT4lDQogIHRhYmxlX21vZGVsdGltZV9hY2N1cmFjeSgNCiAgICAuaW50ZXJhY3RpdmUgPSBGQUxTRQ0KICApDQpgYGANCg0KIyMgUmVmaXQgdG8gRnVsbCBEYXRhc2V0ICYgRm9yZWNhc3QgRm9yd2FyZA0KDQpgYGB7ciByZWZpdCwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCnJlZml0X3RibCA8LSBjYWxpYnJhdGlvbl90YmwgJT4lDQogIG1vZGVsdGltZV9yZWZpdChkYXRhID0gZGYpDQoNCnJlZml0X3RibCAlPiUNCiAgbW9kZWx0aW1lX2ZvcmVjYXN0KGggPSAiMyB5ZWFycyIsIGFjdHVhbF9kYXRhID0gZGYpICU+JQ0KICBwbG90X21vZGVsdGltZV9mb3JlY2FzdCgNCiAgICAubGVnZW5kX21heF93aWR0aCA9IDI1LCAjIEZvciBtb2JpbGUgc2NyZWVucw0KICAgIC5pbnRlcmFjdGl2ZSAgICAgID0gaW50ZXJhY3RpdmUNCiAgKQ0KYGBgICANCg0KIyBNb2RlbCA4OiBFbnNlbWJsZQ0KDQpGYXJlbW9zIGEgcm90aW5hIGRlIGNvbWJpbmHDp8OjbyBkZSBmb3JlY2FzdHMgcGFyYSBvcyBtb2RlbG9zIGF1dG8gQVJJTUEsIFByb3BoZXQsIEVsYXN0aWMgTmV0IGUgRVRTLiANCg0KYGBge3IgbW9kOCwgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCiMgaHR0cHM6Ly9idXNpbmVzcy1zY2llbmNlLmdpdGh1Yi5pby9tb2RlbHRpbWUuZW5zZW1ibGUvYXJ0aWNsZXMvZ2V0dGluZy1zdGFydGVkLXdpdGgtbW9kZWx0aW1lLWVuc2VtYmxlLmh0bWwNCiMgUmVjaXBlDQpyZWNpcGVfc3BlYyA8LSByZWNpcGUodmFsdWUgfiBkYXRlLCB0cmFpbmluZyhzcGxpdHMpKSAlPiUNCiAgICBzdGVwX3RpbWVzZXJpZXNfc2lnbmF0dXJlKGRhdGUpICU+JQ0KICAgIHN0ZXBfcm0obWF0Y2hlcygiKC5pc28kKXwoLnh0cyQpIikpICU+JQ0KICAgIHN0ZXBfbm9ybWFsaXplKG1hdGNoZXMoIihpbmRleC5udW0kKXwoX3llYXIkKSIpKSAlPiUNCiAgICBzdGVwX2R1bW15KGFsbF9ub21pbmFsKCkpICU+JQ0KICAgIHN0ZXBfZm91cmllcihkYXRlLCBLID0gMSwgcGVyaW9kID0gMTIpDQoNCnJlY2lwZV9zcGVjICU+JSBwcmVwKCkgJT4lIGp1aWNlKCkNCiMgTW9kZWwgMSBhdXRvIEFSSU1BDQptb2RlbF9zcGVjX2FyaW1hIDwtIGFyaW1hX3JlZygpICU+JQ0KICAgIHNldF9lbmdpbmUoImF1dG9fYXJpbWEiKQ0KDQp3Zmx3X2ZpdF9hcmltYSA8LSB3b3JrZmxvdygpICU+JQ0KICAgIGFkZF9tb2RlbChtb2RlbF9zcGVjX2FyaW1hKSAlPiUNCiAgICBhZGRfcmVjaXBlKHJlY2lwZV9zcGVjICU+JSBzdGVwX3JtKGFsbF9wcmVkaWN0b3JzKCksIC1kYXRlKSkgJT4lDQogICAgZml0KHRyYWluaW5nKHNwbGl0cykpDQoNCiMgTW9kZWwgMiAtIFByb3BoZXQNCm1vZGVsX3NwZWNfcHJvcGhldCA8LSBwcm9waGV0X3JlZygpICU+JQ0KICAgIHNldF9lbmdpbmUoInByb3BoZXQiKQ0KDQp3Zmx3X2ZpdF9wcm9waGV0IDwtIHdvcmtmbG93KCkgJT4lDQogICAgYWRkX21vZGVsKG1vZGVsX3NwZWNfcHJvcGhldCkgJT4lDQogICAgYWRkX3JlY2lwZShyZWNpcGVfc3BlYyAlPiUgc3RlcF9ybShhbGxfcHJlZGljdG9ycygpLCAtZGF0ZSkpICU+JQ0KICAgIGZpdCh0cmFpbmluZyhzcGxpdHMpKQ0KDQojIE1vZGVsIDMgLSBFbGFzdGljIE5ldA0KbW9kZWxfc3BlY19nbG1uZXQgPC0gbGluZWFyX3JlZygNCiAgICBtaXh0dXJlID0gMC45LA0KICAgIHBlbmFsdHkgPSA0LjM2ZS02ICkgJT4lDQogICAgc2V0X2VuZ2luZSgiZ2xtbmV0IikNCg0Kd2Zsd19maXRfZ2xtbmV0IDwtIHdvcmtmbG93KCkgJT4lDQogICAgYWRkX21vZGVsKG1vZGVsX3NwZWNfZ2xtbmV0KSAlPiUNCiAgICBhZGRfcmVjaXBlKHJlY2lwZV9zcGVjICU+JSBzdGVwX3JtKGRhdGUpKSAlPiUNCiAgICBmaXQodHJhaW5pbmcoc3BsaXRzKSkNCg0KIyBNb2RlbCA0IC0gRVRTDQptb2RlbF9zcGVjX2V0cyA8LSBleHBfc21vb3RoaW5nKCkgJT4lDQogICAgc2V0X2VuZ2luZShlbmdpbmUgPSAiZXRzIikNCiAgDQp3Zmx3X2ZpdF9ldHMgPC0gd29ya2Zsb3coKSAlPiUNCiAgICBhZGRfbW9kZWwobW9kZWxfc3BlY19ldHMpICU+JQ0KICAgIGFkZF9yZWNpcGUocmVjaXBlX3NwZWMgJT4lDQpzdGVwX3JtKGFsbF9wcmVkaWN0b3JzKCksIC1kYXRlKSkgJT4lDQogICAgZml0KHRyYWluaW5nKHNwbGl0cykpDQoNCmBgYA0KDQojIyBTdGVwIDEgLSBDcmVhdGUgYSBNb2RlbHRpbWUgVGFibGUNCg0KYGBge3IgY3JlYXRlX3RhYmxlZW5zLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KYWlycGFzc19tb2RlbHMgPC0gbW9kZWx0aW1lX3RhYmxlKA0KICAgIHdmbHdfZml0X2FyaW1hLA0KICAgIHdmbHdfZml0X3Byb3BoZXQsDQogICAgd2Zsd19maXRfZ2xtbmV0LA0KICAgIHdmbHdfZml0X2V0cw0KKQ0KDQphaXJwYXNzX21vZGVscw0KYGBgDQoNCg0KIyMgU3RlcCAyIC0gTWFrZSBhbiBFbnNlbWJsZQ0KDQpgYGB7ciBtYWtlX2VucywgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCmVuc2VtYmxlX2ZpdCA8LSBhaXJwYXNzX21vZGVscyAlPiUNCiAgICBlbnNlbWJsZV9hdmVyYWdlKHR5cGUgPSAibWVhbiIpDQoNCmVuc2VtYmxlX2ZpdA0KYGBgDQoNCg0KIyMgU3RlcCAzIC0gRm9yZWNhc3QhICh0aGUgVGVzdCBEYXRhKQ0KDQpgYGB7ciBmY3N0X2VucywgbWVzc2FnZT1GQUxTRSwgd2FybmluZz1GQUxTRX0NCiMgQ2FsaWJyYXRpb24NCmNhbGlicmF0aW9uX3RibCA8LSBtb2RlbHRpbWVfdGFibGUoDQogICAgZW5zZW1ibGVfZml0DQopICU+JQ0KICAgIG1vZGVsdGltZV9jYWxpYnJhdGUodGVzdGluZyhzcGxpdHMpKQ0KDQojIEZvcmVjYXN0IHZzIFRlc3QgU2V0DQpjYWxpYnJhdGlvbl90YmwgJT4lDQogICAgbW9kZWx0aW1lX2ZvcmVjYXN0KA0KICAgICAgICBuZXdfZGF0YSAgICA9IHRlc3Rpbmcoc3BsaXRzKSwNCiAgICAgICAgYWN0dWFsX2RhdGEgPSBkZg0KICAgICkgJT4lDQogICAgcGxvdF9tb2RlbHRpbWVfZm9yZWNhc3QoLmludGVyYWN0aXZlID0gVCkNCmBgYA0KDQojIyBTdGVwIDQgLSBSZWZpdCBvbiBGdWxsIERhdGEgJiBGb3JlY2FzdCBGdXR1cmUNCg0KYGBge3IgYWNjdV9lbnMsIG1lc3NhZ2U9RkFMU0UsIHdhcm5pbmc9RkFMU0V9DQpjYWxpYnJhdGlvbl90YmwgJT4lDQogIG1vZGVsdGltZV9hY2N1cmFjeSgpICU+JQ0KICB0YWJsZV9tb2RlbHRpbWVfYWNjdXJhY3koDQogICAgLmludGVyYWN0aXZlID0gVA0KICApDQpgYGANCg0KYGBge3IgY2FsaWJfZW5zLCBtZXNzYWdlPUZBTFNFLCB3YXJuaW5nPUZBTFNFfQ0KcmVmaXRfdGJsIDwtIGNhbGlicmF0aW9uX3RibCAlPiUNCiAgICBtb2RlbHRpbWVfcmVmaXQoZGYpDQoNCnJlZml0X3RibCAlPiUNCiAgICBtb2RlbHRpbWVfZm9yZWNhc3QoDQogICAgICAgIGggPSAiMiB5ZWFycyIsDQogICAgICAgIGFjdHVhbF9kYXRhID0gZGYNCiAgICApICU+JQ0KICAgIHBsb3RfbW9kZWx0aW1lX2ZvcmVjYXN0KC5pbnRlcmFjdGl2ZSA9IFQpDQpgYGANCg0KIyBSZWZlcsOqbmNpYXMgeyNSZWZlcsOqbmNpYXMgLnVubnVtYmVyZWR9DQoNCkRhbmNobyBNICgyMDIzKS4gX21vZGVsdGltZTogVGhlIFRpZHltb2RlbHMgRXh0ZW5zaW9uIGZvciBUaW1lIFNlcmllcyBNb2RlbGluZ18uIFIgcGFja2FnZSB2ZXJzaW9uIDEuMi42LCA8aHR0cHM6Ly9DUkFOLlItcHJvamVjdC5vcmcvcGFja2FnZT1tb2RlbHRpbWU+Lg0K