Load necessary libraries

options(warnPartialMatchArgs = FALSE)  # don't want these warnings
library(magrittr)     # pipes
library(tibble)       # tibbles
library(dplyr)        # data wrangling
library(boot)         # for `cv.glm`
library(purrr)        # iteration
library(ggplot2)      # tidy plotting
library(ISLR)         # Auto data set

1 The Auto Data Set

Auto %<>% as.tibble()                     # Convert -> tibble
class(Auto)                               # Confirm Auto is now a tibble
[1] "tbl_df"     "tbl"        "data.frame"
names(Auto)                               # List variables
[1] "mpg"          "cylinders"    "displacement" "horsepower"   "weight"       "acceleration"
[7] "year"         "origin"       "name"        
Auto                                      # Print tibble; top 10 by default
Auto %>%                                  
  ggplot(aes(x = horsepower, y = mpg)) +  # Plot hp vs mpg
  geom_point(alpha = 0.5) +               # Create a scatter plot
  geom_smooth(method = 'loess')           # Fit a polynomial smoothing function

2 Cross-validation (by hand)

# Create a random partition into training and test sets
set.seed(1)
n     <- nrow(Auto)
train <- sample(1:n, n/2)         # Randomly select half of the numbers 1 to n

# Fit linear model -> mpg as a function of hp
stats::lm(mpg ~ horsepower, data = Auto, subset = train)  # linear in horsepower

Call:
stats::lm(formula = mpg ~ horsepower, data = Auto, subset = train)

Coefficients:
(Intercept)   horsepower  
    40.3404      -0.1617  
# Loop over polynomial power and calculate MSE on test set (-train)
purrr::map_dbl(1:5,
  ~stats::lm(mpg ~ stats::poly(horsepower, .x),  # fit lm
             data = Auto, subset = train) %>%  
    predict(., Auto) %>%                 # Predict for all samples
    magrittr::subtract(Auto$mpg) %>%     # Obtain differences from actual values
    magrittr::raise_to_power(2) %>%      # Square differences
    magrittr::extract(-train) %>%        # Ignore training predictions
    mean()                               # MSE
)
[1] 26.14142 19.82259 19.78252 19.99969 20.18225
# Set a different random seed to create a different random train/test partition
set.seed(2)
train <- sample(1:n, n/2)                # Randomly select half of the numbers 1 to n
purrr::map_dbl(1:5,
  ~stats::lm(mpg ~ stats::poly(horsepower, .x),
             data = Auto, subset = train) %>%  # fit lm
    predict(., Auto) %>%                 # Predict for all samples
    magrittr::subtract(Auto$mpg) %>%     # Obtain differences from actual values
    magrittr::raise_to_power(2) %>%      # Square differences
    magrittr::extract(-train) %>%        # Ignore training predictions
    mean()                               # MSE
)
[1] 23.29559 18.90124 19.25740 20.38538 20.29775

3 Leave-One-Out CV (boot::cv.glm)

# Linear regression using glm
glm_fit <- stats::glm(mpg ~ horsepower, data = Auto)
cv_err  <- boot::cv.glm(Auto, glm_fit)
cv_err$K            # Default K = n; i.e. LOOCV
[1] 392
cv_err$delta
[1] 24.23151 24.23114
# Loop over polynomial power
# raw = TRUE issue resolved
purrr::map_dbl(1:5, function(.x) {
  fit <- stats::glm(mpg ~ stats::poly(horsepower, degree = .x, raw = TRUE),
                    data = Auto)
  boot::cv.glm(Auto, fit) %>%
  purrr::pluck("delta") %>%     # Retrieve the `delta` element
  purrr::pluck(1)               # Extract the first component of `delta`
})
[1] 24.23151 19.24821 19.33498 19.42443 19.03321

4 K-fold CV (boot::cv.glm)

set.seed(17)
# K-folds chosen at random; here K = 10
# Loop over polynomial power
# Default: `raw = FALSE`
purrr::map_dbl(1:10, function(.x) {
  fit <- stats::glm(mpg ~ stats::poly(horsepower, degree = .x), data = Auto)
  boot::cv.glm(Auto, fit, K = 10) %>%
    purrr::pluck("delta") %>%
    purrr::pluck(1)
})
 [1] 169.3269 190.3656 194.8383 195.6951 192.9770 196.7003 197.5728 197.6055 195.7763
[10] 203.9586

5 Bootstrap (boot::boot)

Here we estimate the accuracy of a linear regression model via the bootstrap. We will first define a function to fit a linear model of mpg ~ horsepower given a dataset and a subset index (rows).

5.1 Sampling with replacement

# What does sample with replacement do?
# Sampling with replacement is what the bootstrap is all about!
set.seed(3)
sample(1:10, 10, replace = FALSE)
 [1]  2  8  4  3  9  6  1  5 10  7
sample(1:10, 10, replace = TRUE)
 [1] 6 6 6 6 9 9 2 8 9 3

5.2 Create a boot function

boot_fn <- function(data, index) {
  lm(mpg ~ horsepower, data = data, subset = index) %>%
    purrr::pluck("coefficients")
}

5.3 Perform bootstrap fits (by hand)

# Using all the data
boot_fn(Auto, 1:392)
(Intercept)  horsepower 
 39.9358610  -0.1578447 
# Make the simulation reproducible
set.seed(1)

# First sample
sample(1:392, 392, replace = TRUE) %>% boot_fn(Auto, .)
(Intercept)  horsepower 
 38.7387134  -0.1481952 
# Second sample
sample(1:392, 392, replace = TRUE) %>% boot_fn(Auto, .)
(Intercept)  horsepower 
 40.0383086  -0.1596104

5.4 Use boot::boot()

# 1000 bootstrap samples; gets tedious, so use `boot::boot`
boot::boot(Auto, boot_fn, R = 1000)

ORDINARY NONPARAMETRIC BOOTSTRAP


Call:
boot::boot(data = Auto, statistic = boot_fn, R = 1000)


Bootstrap Statistics :
      original      bias    std. error
t1* 39.9358610  0.02972191 0.860007896
t2* -0.1578447 -0.00030823 0.007404467
# Compare with linear regression
# (differences indicate some assumptions may be broken)
lm(mpg ~ horsepower, data = Auto) %>%
  summary() %>%
  purrr::pluck("coefficients")
              Estimate  Std. Error   t value      Pr(>|t|)
(Intercept) 39.9358610 0.717498656  55.65984 1.220362e-187
horsepower  -0.1578447 0.006445501 -24.48914  7.031989e-81

5.5 Repeat with a quadratic model

# Redefine `boot_fn()`
boot_fn2 <- function(data, index) {
  lm(mpg ~ horsepower + I(horsepower^2), data = data, subset = index) %>%
    purrr::pluck("coefficients")
}

# Make the simulation reproducible
set.seed(1)
boot(Auto, boot_fn2, 1000)

ORDINARY NONPARAMETRIC BOOTSTRAP


Call:
boot(data = Auto, statistic = boot_fn2, R = 1000)


Bootstrap Statistics :
        original        bias     std. error
t1* 56.900099702  6.098115e-03 2.0944855842
t2* -0.466189630 -1.777108e-04 0.0334123802
t3*  0.001230536  1.324315e-06 0.0001208339
# Compare with standard error from linear regression
lm(mpg ~ horsepower + I(horsepower^2), data = Auto) %>%
  summary() %>%
  purrr::pluck("coefficients")
                    Estimate   Std. Error   t value      Pr(>|t|)
(Intercept)     56.900099702 1.8004268063  31.60367 1.740911e-109
horsepower      -0.466189630 0.0311246171 -14.97816  2.289429e-40
I(horsepower^2)  0.001230536 0.0001220759  10.08009  2.196340e-21

Created on 2020-01-30 by Rmarkdown (v1.10) and R version 3.5.0 (2018-04-23).

LS0tDQp0aXRsZTogJ1NUQUEgNTc3OiBMYWJvcmF0b3J5IEZpdmUgPC9icj4gUmVzYW1wbGluZyBtZXRob2RzIChgdGlkeXZlcnNlYCknDQphdXRob3I6ICdBZGFwdGVkIGJ5IFRhdmVuZXIgJiBGaWVsZCA8L2JyPiBGcm9tOiBKYW1lcywgV2l0dGVuLCBIYXN0aWUgYW5kIFRpYnNoaXJhbmknDQpkYXRlOiAiYHIgZm9ybWF0KFN5cy5EYXRlKCksICclZSAlQiAlWScpYCINCm91dHB1dDoNCiAgaHRtbF9ub3RlYm9vazoNCiAgICBjb2RlX2ZvbGRpbmc6IHNob3cNCiAgICBudW1iZXJfc2VjdGlvbnM6IHllcw0KICAgIHRvYzogeWVzDQogICAgdG9jX2Zsb2F0Og0KICAgICAgY29sbGFwc2VkOiBubw0KcmF0aW86ICc5OjE2Jw0KdGFibGVzOiB5ZXMNCmZvbnRzaXplOiAxMnB0DQotLS0NCg0KDQojIyMgTG9hZCBuZWNlc3NhcnkgbGlicmFyaWVzIHstfQ0KYGBge3Igc2V0dXAsIG1lc3NhZ2UgPSBGQUxTRSwgd2FybmluZyA9IEZBTFNFfQ0Kb3B0aW9ucyh3YXJuUGFydGlhbE1hdGNoQXJncyA9IEZBTFNFKSAgIyBkb24ndCB3YW50IHRoZXNlIHdhcm5pbmdzDQpsaWJyYXJ5KG1hZ3JpdHRyKSAgICAgIyBwaXBlcw0KbGlicmFyeSh0aWJibGUpICAgICAgICMgdGliYmxlcw0KbGlicmFyeShkcGx5cikgICAgICAgICMgZGF0YSB3cmFuZ2xpbmcNCmxpYnJhcnkoYm9vdCkgICAgICAgICAjIGZvciBgY3YuZ2xtYA0KbGlicmFyeShwdXJycikgICAgICAgICMgaXRlcmF0aW9uDQpsaWJyYXJ5KGdncGxvdDIpICAgICAgIyB0aWR5IHBsb3R0aW5nDQpsaWJyYXJ5KElTTFIpICAgICAgICAgIyBBdXRvIGRhdGEgc2V0DQpgYGANCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCiMgVGhlIGBBdXRvYCBEYXRhIFNldA0KYGBge3IgZGF0YSwgaW5jbHVkZSA9IFRSVUV9DQpBdXRvICU8PiUgYXMudGliYmxlKCkgICAgICAgICAgICAgICAgICAgICAjIENvbnZlcnQgLT4gdGliYmxlDQpjbGFzcyhBdXRvKSAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIENvbmZpcm0gQXV0byBpcyBub3cgYSB0aWJibGUNCm5hbWVzKEF1dG8pICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICMgTGlzdCB2YXJpYWJsZXMNCkF1dG8gICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICMgUHJpbnQgdGliYmxlOyB0b3AgMTAgYnkgZGVmYXVsdA0KQXV0byAlPiUgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgDQogIGdncGxvdChhZXMoeCA9IGhvcnNlcG93ZXIsIHkgPSBtcGcpKSArICAjIFBsb3QgaHAgdnMgbXBnDQogIGdlb21fcG9pbnQoYWxwaGEgPSAwLjUpICsgICAgICAgICAgICAgICAjIENyZWF0ZSBhIHNjYXR0ZXIgcGxvdA0KICBnZW9tX3Ntb290aChtZXRob2QgPSAnbG9lc3MnKSAgICAgICAgICAgIyBGaXQgYSBwb2x5bm9taWFsIHNtb290aGluZyBmdW5jdGlvbg0KYGBgDQoNCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCg0KIyBDcm9zcy12YWxpZGF0aW9uIChieSBoYW5kKQ0KDQpgYGB7ciBjdn0NCiMgQ3JlYXRlIGEgcmFuZG9tIHBhcnRpdGlvbiBpbnRvIHRyYWluaW5nIGFuZCB0ZXN0IHNldHMNCnNldC5zZWVkKDEpDQpuICAgICA8LSBucm93KEF1dG8pDQp0cmFpbiA8LSBzYW1wbGUoMTpuLCBuLzIpICAgICAgICAgIyBSYW5kb21seSBzZWxlY3QgaGFsZiBvZiB0aGUgbnVtYmVycyAxIHRvIG4NCg0KIyBGaXQgbGluZWFyIG1vZGVsIC0+IG1wZyBhcyBhIGZ1bmN0aW9uIG9mIGhwDQpzdGF0czo6bG0obXBnIH4gaG9yc2Vwb3dlciwgZGF0YSA9IEF1dG8sIHN1YnNldCA9IHRyYWluKSAgIyBsaW5lYXIgaW4gaG9yc2Vwb3dlcg0KDQojIExvb3Agb3ZlciBwb2x5bm9taWFsIHBvd2VyIGFuZCBjYWxjdWxhdGUgTVNFIG9uIHRlc3Qgc2V0ICgtdHJhaW4pDQpwdXJycjo6bWFwX2RibCgxOjUsDQogIH5zdGF0czo6bG0obXBnIH4gc3RhdHM6OnBvbHkoaG9yc2Vwb3dlciwgLngpLCAgIyBmaXQgbG0NCiAgICAgICAgICAgICBkYXRhID0gQXV0bywgc3Vic2V0ID0gdHJhaW4pICU+JSAgDQogICAgcHJlZGljdCguLCBBdXRvKSAlPiUgICAgICAgICAgICAgICAgICMgUHJlZGljdCBmb3IgYWxsIHNhbXBsZXMNCiAgICBtYWdyaXR0cjo6c3VidHJhY3QoQXV0byRtcGcpICU+JSAgICAgIyBPYnRhaW4gZGlmZmVyZW5jZXMgZnJvbSBhY3R1YWwgdmFsdWVzDQogICAgbWFncml0dHI6OnJhaXNlX3RvX3Bvd2VyKDIpICU+JSAgICAgICMgU3F1YXJlIGRpZmZlcmVuY2VzDQogICAgbWFncml0dHI6OmV4dHJhY3QoLXRyYWluKSAlPiUgICAgICAgICMgSWdub3JlIHRyYWluaW5nIHByZWRpY3Rpb25zDQogICAgbWVhbigpICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICMgTVNFDQopDQoNCiMgU2V0IGEgZGlmZmVyZW50IHJhbmRvbSBzZWVkIHRvIGNyZWF0ZSBhIGRpZmZlcmVudCByYW5kb20gdHJhaW4vdGVzdCBwYXJ0aXRpb24NCnNldC5zZWVkKDIpDQp0cmFpbiA8LSBzYW1wbGUoMTpuLCBuLzIpICAgICAgICAgICAgICAgICMgUmFuZG9tbHkgc2VsZWN0IGhhbGYgb2YgdGhlIG51bWJlcnMgMSB0byBuDQpwdXJycjo6bWFwX2RibCgxOjUsDQogIH5zdGF0czo6bG0obXBnIH4gc3RhdHM6OnBvbHkoaG9yc2Vwb3dlciwgLngpLA0KICAgICAgICAgICAgIGRhdGEgPSBBdXRvLCBzdWJzZXQgPSB0cmFpbikgJT4lICAjIGZpdCBsbQ0KICAgIHByZWRpY3QoLiwgQXV0bykgJT4lICAgICAgICAgICAgICAgICAjIFByZWRpY3QgZm9yIGFsbCBzYW1wbGVzDQogICAgbWFncml0dHI6OnN1YnRyYWN0KEF1dG8kbXBnKSAlPiUgICAgICMgT2J0YWluIGRpZmZlcmVuY2VzIGZyb20gYWN0dWFsIHZhbHVlcw0KICAgIG1hZ3JpdHRyOjpyYWlzZV90b19wb3dlcigyKSAlPiUgICAgICAjIFNxdWFyZSBkaWZmZXJlbmNlcw0KICAgIG1hZ3JpdHRyOjpleHRyYWN0KC10cmFpbikgJT4lICAgICAgICAjIElnbm9yZSB0cmFpbmluZyBwcmVkaWN0aW9ucw0KICAgIG1lYW4oKSAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAjIE1TRQ0KKQ0KYGBgDQoNCg0KDQojIExlYXZlLU9uZS1PdXQgQ1YgKGBib290Ojpjdi5nbG1gKQ0KDQpgYGB7ciBMT09DVn0NCiMgTGluZWFyIHJlZ3Jlc3Npb24gdXNpbmcgZ2xtDQpnbG1fZml0IDwtIHN0YXRzOjpnbG0obXBnIH4gaG9yc2Vwb3dlciwgZGF0YSA9IEF1dG8pDQpjdl9lcnIgIDwtIGJvb3Q6OmN2LmdsbShBdXRvLCBnbG1fZml0KQ0KY3ZfZXJyJEsgICAgICAgICAgICAjIERlZmF1bHQgSyA9IG47IGkuZS4gTE9PQ1YNCmN2X2VyciRkZWx0YQ0KYGBgDQoNCg0KYGBge3IgTE9PQ1ZfcG9seSwgZXZhbCA9IEZBTFNFLCBlY2hvID0gRkFMU0V9DQojIExvb3Agb3ZlciBwb2x5bm9taWFsIHBvd2VyDQojIFRoaXMgY2h1bmsgbm90IGN1cnJlbnRseSBydW4NCnB1cnJyOjptYXBfZGJsKDE6NSwgZnVuY3Rpb24oLngpIHsNCiAgZml0IDwtIHN0YXRzOjpnbG0obXBnIH4gc3RhdHM6OnBvbHkoaG9yc2Vwb3dlciwgZGVncmVlID0gLngpLA0KICAgICAgICAgICAgICAgICAgICBkYXRhID0gQXV0bykNCiAgYm9vdDo6Y3YuZ2xtKEF1dG8sIGZpdCkgJT4lDQogIHB1cnJyOjpwbHVjaygiZGVsdGEiKSAlPiUgICAgICMgUmV0cmlldmUgdGhlIGBkZWx0YWAgZWxlbWVudA0KICBwdXJycjo6cGx1Y2soMSkgICAgICAgICAgICAgICAjIEV4dHJhY3QgdGhlIGZpcnN0IGNvbXBvbmVudCBvZiBgZGVsdGFgDQp9KQ0KYGBgDQoNCg0KYGBge3IgTE9PQ1ZfcG9seV8yfQ0KIyBMb29wIG92ZXIgcG9seW5vbWlhbCBwb3dlcg0KIyByYXcgPSBUUlVFIGlzc3VlIHJlc29sdmVkDQpwdXJycjo6bWFwX2RibCgxOjUsIGZ1bmN0aW9uKC54KSB7DQogIGZpdCA8LSBzdGF0czo6Z2xtKG1wZyB+IHN0YXRzOjpwb2x5KGhvcnNlcG93ZXIsIGRlZ3JlZSA9IC54LCByYXcgPSBUUlVFKSwNCiAgICAgICAgICAgICAgICAgICAgZGF0YSA9IEF1dG8pDQogIGJvb3Q6OmN2LmdsbShBdXRvLCBmaXQpICU+JQ0KICBwdXJycjo6cGx1Y2soImRlbHRhIikgJT4lICAgICAjIFJldHJpZXZlIHRoZSBgZGVsdGFgIGVsZW1lbnQNCiAgcHVycnI6OnBsdWNrKDEpICAgICAgICAgICAgICAgIyBFeHRyYWN0IHRoZSBmaXJzdCBjb21wb25lbnQgb2YgYGRlbHRhYA0KfSkNCmBgYA0KDQoNCi0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0NCg0KDQojIEstZm9sZCBDViAoYGJvb3Q6OmN2LmdsbWApDQoNCmBgYHtyIGstZm9sZF9jcm9zc192YWxpZGF0aW9uX29ydGhvZ29uYWxfcG9seW5vbWlhbHN9DQpzZXQuc2VlZCgxNykNCiMgSy1mb2xkcyBjaG9zZW4gYXQgcmFuZG9tOyBoZXJlIEsgPSAxMA0KIyBMb29wIG92ZXIgcG9seW5vbWlhbCBwb3dlcg0KIyBEZWZhdWx0OiBgcmF3ID0gRkFMU0VgDQpwdXJycjo6bWFwX2RibCgxOjEwLCBmdW5jdGlvbigueCkgew0KICBmaXQgPC0gc3RhdHM6OmdsbShtcGcgfiBzdGF0czo6cG9seShob3JzZXBvd2VyLCBkZWdyZWUgPSAueCksIGRhdGEgPSBBdXRvKQ0KICBib290Ojpjdi5nbG0oQXV0bywgZml0LCBLID0gMTApICU+JQ0KICAgIHB1cnJyOjpwbHVjaygiZGVsdGEiKSAlPiUNCiAgICBwdXJycjo6cGx1Y2soMSkNCn0pDQpgYGANCg0KLS0tLS0tLS0tLS0tLS0tLS0tLS0NCg0KIyBCb290c3RyYXAgKGBib290Ojpib290YCkNCg0KSGVyZSB3ZSBlc3RpbWF0ZSB0aGUgYWNjdXJhY3kgb2YgYSBsaW5lYXIgcmVncmVzc2lvbiBtb2RlbCB2aWEgdGhlIGJvb3RzdHJhcC4NCldlIHdpbGwgZmlyc3QgZGVmaW5lIGEgZnVuY3Rpb24gdG8gZml0IGEgbGluZWFyIG1vZGVsIG9mIGBtcGcgfiBob3JzZXBvd2VyYA0KZ2l2ZW4gYSBkYXRhc2V0IGFuZCBhIHN1YnNldCBpbmRleCAocm93cykuDQoNCiMjIFNhbXBsaW5nIHdpdGggcmVwbGFjZW1lbnQNCmBgYHtyIFNhbXBsaW5nX3dpdGhfcmVwbGFjZW1lbnR9DQojIFdoYXQgZG9lcyBzYW1wbGUgd2l0aCByZXBsYWNlbWVudCBkbz8NCiMgU2FtcGxpbmcgd2l0aCByZXBsYWNlbWVudCBpcyB3aGF0IHRoZSBib290c3RyYXAgaXMgYWxsIGFib3V0IQ0Kc2V0LnNlZWQoMykNCnNhbXBsZSgxOjEwLCAxMCwgcmVwbGFjZSA9IEZBTFNFKQ0Kc2FtcGxlKDE6MTAsIDEwLCByZXBsYWNlID0gVFJVRSkNCmBgYA0KDQojIyBDcmVhdGUgYSBib290IGZ1bmN0aW9uDQpgYGB7ciBib290c3RyYXBfZnVuY3Rpb259DQpib290X2ZuIDwtIGZ1bmN0aW9uKGRhdGEsIGluZGV4KSB7DQogIGxtKG1wZyB+IGhvcnNlcG93ZXIsIGRhdGEgPSBkYXRhLCBzdWJzZXQgPSBpbmRleCkgJT4lDQogICAgcHVycnI6OnBsdWNrKCJjb2VmZmljaWVudHMiKQ0KfQ0KYGBgDQoNCiMjIFBlcmZvcm0gYm9vdHN0cmFwIGZpdHMgKGJ5IGhhbmQpDQpgYGB7ciBib290c3RyYXB9DQojIFVzaW5nIGFsbCB0aGUgZGF0YQ0KYm9vdF9mbihBdXRvLCAxOjM5MikNCg0KIyBNYWtlIHRoZSBzaW11bGF0aW9uIHJlcHJvZHVjaWJsZQ0Kc2V0LnNlZWQoMSkNCg0KIyBGaXJzdCBzYW1wbGUNCnNhbXBsZSgxOjM5MiwgMzkyLCByZXBsYWNlID0gVFJVRSkgJT4lIGJvb3RfZm4oQXV0bywgLikNCg0KIyBTZWNvbmQgc2FtcGxlDQpzYW1wbGUoMTozOTIsIDM5MiwgcmVwbGFjZSA9IFRSVUUpICU+JSBib290X2ZuKEF1dG8sIC4pDQpgYGANCg0KDQojIyBVc2UgYGJvb3Q6OmJvb3QoKWANCmBgYHtyIGJvb3RzdHJhcCB3aXRoIGJvb3Q6OmJvb3R9DQojIDEwMDAgYm9vdHN0cmFwIHNhbXBsZXM7IGdldHMgdGVkaW91cywgc28gdXNlIGBib290Ojpib290YA0KYm9vdDo6Ym9vdChBdXRvLCBib290X2ZuLCBSID0gMTAwMCkNCg0KIyBDb21wYXJlIHdpdGggbGluZWFyIHJlZ3Jlc3Npb24NCiMgKGRpZmZlcmVuY2VzIGluZGljYXRlIHNvbWUgYXNzdW1wdGlvbnMgbWF5IGJlIGJyb2tlbikNCmxtKG1wZyB+IGhvcnNlcG93ZXIsIGRhdGEgPSBBdXRvKSAlPiUNCiAgc3VtbWFyeSgpICU+JQ0KICBwdXJycjo6cGx1Y2soImNvZWZmaWNpZW50cyIpDQpgYGANCg0KIyMgUmVwZWF0IHdpdGggYSBxdWFkcmF0aWMgbW9kZWwNCg0KYGBge3IgYm9vdDJ9DQojIFJlZGVmaW5lIGBib290X2ZuKClgDQpib290X2ZuMiA8LSBmdW5jdGlvbihkYXRhLCBpbmRleCkgew0KICBsbShtcGcgfiBob3JzZXBvd2VyICsgSShob3JzZXBvd2VyXjIpLCBkYXRhID0gZGF0YSwgc3Vic2V0ID0gaW5kZXgpICU+JQ0KICAgIHB1cnJyOjpwbHVjaygiY29lZmZpY2llbnRzIikNCn0NCg0KIyBNYWtlIHRoZSBzaW11bGF0aW9uIHJlcHJvZHVjaWJsZQ0Kc2V0LnNlZWQoMSkNCmJvb3QoQXV0bywgYm9vdF9mbjIsIDEwMDApDQoNCiMgQ29tcGFyZSB3aXRoIHN0YW5kYXJkIGVycm9yIGZyb20gbGluZWFyIHJlZ3Jlc3Npb24NCmxtKG1wZyB+IGhvcnNlcG93ZXIgKyBJKGhvcnNlcG93ZXJeMiksIGRhdGEgPSBBdXRvKSAlPiUNCiAgc3VtbWFyeSgpICU+JQ0KICBwdXJycjo6cGx1Y2soImNvZWZmaWNpZW50cyIpDQpgYGANCg0KDQotLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQoNCkNyZWF0ZWQgb24gYHIgU3lzLkRhdGUoKWAgYnkgW1JtYXJrZG93bl0oaHR0cHM6Ly9naXRodWIuY29tL3JzdHVkaW8vcm1hcmtkb3duKQ0KKHZgciB1dGlsczo6cGFja2FnZVZlcnNpb24oInJtYXJrZG93biIpYCkgYW5kIGByIFIudmVyc2lvbiR2ZXJzaW9uLnN0cmluZ2AuDQo=