Hedonic Pricing Models with Machine Learning
Start of Code
Load packages
Read in Data
# Read in data
library(readxl)
data_gbm_clean <- read_excel("/Users/sawyerbenson/Documents/Master Thesis/Thesis_Github/Models/Data/New Data/3. data_factor_cleaned.xlsx")
# Structure Change
data_gbm_clean$property_type <- as.factor(data_gbm_clean$property_type)
data_gbm_clean$ac_type <- as.factor(data_gbm_clean$ac_type)
data_gbm_clean$patio <- as.factor(data_gbm_clean$patio)
data_gbm_clean$school_general <- as.factor(data_gbm_clean$school_general)
data_gbm_clean$pool <- as.factor(data_gbm_clean$pool)
data_gbm_clean$roof_type <- as.factor(data_gbm_clean$roof_type)
data_gbm_clean$gas_type <- as.factor(data_gbm_clean$gas_type)
data_gbm_clean$out_building <- as.factor(data_gbm_clean$out_building)
data_gbm_clean$appliances <- as.factor(data_gbm_clean$appliances)
data_gbm_clean$garage <- as.factor(data_gbm_clean$garage)
data_gbm_clean$property_condition <- as.factor(data_gbm_clean$property_condition)
data_gbm_clean$energy_efficient <- as.factor(data_gbm_clean$energy_efficient)
data_gbm_clean$exterior_type <- as.factor(data_gbm_clean$exterior_type)
data_gbm_clean$exterior_features <- as.factor(data_gbm_clean$exterior_features)
data_gbm_clean$fireplace <- as.factor(data_gbm_clean$fireplace)
data_gbm_clean$foundation_type <- as.factor(data_gbm_clean$foundation_type)
data_gbm_clean$beds_total <- as.factor(data_gbm_clean$beds_total)
data_gbm_clean$bath_full <- as.factor(data_gbm_clean$bath_full)
data_gbm_clean$bath_half <- as.factor(data_gbm_clean$bath_half)
data_gbm_clean$sewer_type <- as.factor(data_gbm_clean$sewer_type)
data_gbm_clean$property_style <- as.factor(data_gbm_clean$property_style)
data_gbm_clean$subdivision <- as.factor(data_gbm_clean$subdivision)
data_gbm_clean$water_type <- as.factor(data_gbm_clean$water_type)
data_gbm_clean$waterfront <- as.factor(data_gbm_clean$waterfront)
data_gbm_clean$sold_date <- openxlsx::convertToDate(data_gbm_clean$sold_date)
data_gbm_clean$sold_date <- as.numeric(data_gbm_clean$sold_date)
str(data_gbm_clean)tibble [24,412 × 48] (S3: tbl_df/tbl/data.frame)
$ mls_number : chr [1:24412] "CNNN5274" "CNNN5241" "CNN104918" "CNN104870" ...
$ property_type : Factor w/ 6 levels "CND","DUP","OTH",..: 5 5 5 5 5 5 5 5 5 5 ...
$ ac_type : Factor w/ 3 levels "central","none",..: 1 3 1 1 1 1 1 1 1 1 ...
$ list_price : num [1:24412] 187000 250000 224900 225000 274900 ...
$ patio : Factor w/ 2 levels "0","1": 1 1 1 2 2 1 2 2 2 2 ...
$ school_general : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ photo_count : num [1:24412] 0 0 0 0 25 2 6 17 17 15 ...
$ pool : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 2 1 2 1 ...
$ roof_type : Factor w/ 4 levels "metal","other",..: 3 3 2 3 3 3 2 3 2 2 ...
$ gas_type : Factor w/ 5 levels "butane","natural",..: 5 5 5 5 5 5 5 5 5 5 ...
$ out_building : Factor w/ 2 levels "0","1": 1 1 2 1 1 1 1 1 2 1 ...
$ area_living : num [1:24412] 2054 2120 2078 1923 2184 ...
$ land_acres : num [1:24412] 0.28 0.4 0.29 0.36 0.82 0.36 1 1.27 0.63 2.01 ...
$ appliances : Factor w/ 2 levels "0","1": 2 1 2 2 2 2 2 2 2 2 ...
$ garage : Factor w/ 2 levels "0","1": 2 2 1 2 2 2 2 2 2 2 ...
$ property_condition : Factor w/ 3 levels "excellent","new",..: 3 3 3 3 3 3 3 3 3 3 ...
$ energy_efficient : Factor w/ 2 levels "0","1": 1 1 1 1 2 1 1 2 1 2 ...
$ exterior_type : Factor w/ 5 levels "brick","metal",..: 3 3 4 4 1 3 4 1 3 3 ...
$ exterior_features : Factor w/ 6 levels "balcony","courtyard",..: 4 4 4 5 5 3 4 5 3 3 ...
$ fireplace : Factor w/ 2 levels "0","1": 2 1 1 2 2 2 2 2 2 2 ...
$ foundation_type : Factor w/ 3 levels "raised","slab",..: 1 2 1 2 2 3 2 2 2 2 ...
$ area_total : num [1:24412] 2254 2120 2962 2550 3510 ...
$ beds_total : Factor w/ 7 levels "0","1","2","3",..: 4 5 4 4 4 4 5 4 5 5 ...
$ bath_full : Factor w/ 7 levels "0","1","2","3",..: 3 3 3 3 3 3 3 3 3 3 ...
$ bath_half : Factor w/ 6 levels "0","1","2","3",..: 1 1 1 2 1 1 1 1 2 1 ...
$ age : num [1:24412] 82 9 70 27 7 6 38 32 15 5 ...
$ dom : num [1:24412] 78 83 89 203 231 54 144 108 26 25 ...
$ sold_price : num [1:24412] 169000 245000 230000 220000 272000 ...
$ sold_date : num [1:24412] 16843 17123 17228 17336 17324 ...
$ sewer_type : Factor w/ 3 levels "city","septic",..: 1 1 1 2 2 1 3 3 1 3 ...
$ property_style : Factor w/ 2 levels "mobile","not_mobile": 2 2 2 2 2 2 2 2 2 2 ...
$ city_limits : num [1:24412] 1 1 1 1 1 1 1 1 1 1 ...
$ subdivision : Factor w/ 2 levels "0","1": 2 1 2 2 2 2 2 2 2 2 ...
$ water_type : Factor w/ 2 levels "public","well": 1 1 1 1 1 1 1 1 1 1 ...
$ waterfront : Factor w/ 2 levels "0","1": 1 1 1 2 1 1 1 1 1 1 ...
$ infections_daily : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ infections_accum : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ corona_date_split : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ infections_3mma : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ top25_sold_price : num [1:24412] 0 1 0 0 1 1 0 1 1 1 ...
$ top50_sold_price : num [1:24412] 0 1 1 1 1 1 1 1 1 1 ...
$ bottom25_sold_price : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ top25_area_living : num [1:24412] 1 1 1 0 1 1 1 1 1 0 ...
$ bottom25_area_living: num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ top25_age : num [1:24412] 1 0 1 0 0 0 0 0 0 0 ...
$ bottom25_age : num [1:24412] 0 1 0 0 1 1 0 0 1 1 ...
$ top25_dom : num [1:24412] 0 0 0 1 1 0 1 0 0 0 ...
$ bottom25_dom : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...# Splits
data_gbm_clean$city_limits <- as.factor(data_gbm_clean$city_limits)
data_gbm_clean$corona_date_split <- as.factor(data_gbm_clean$corona_date_split)
data_gbm_clean$top25_sold_price <- as.factor(data_gbm_clean$top25_sold_price)
data_gbm_clean$bottom25_sold_price <- as.factor(data_gbm_clean$bottom25_sold_price)
data_gbm_clean$top25_area_living <- as.factor(data_gbm_clean$top25_area_living)
data_gbm_clean$bottom25_area_living <- as.factor(data_gbm_clean$bottom25_area_living)
data_gbm_clean$top25_age <- as.factor(data_gbm_clean$top25_age)
data_gbm_clean$bottom25_age <- as.factor(data_gbm_clean$bottom25_age)
data_gbm_clean$top25_dom <- as.factor(data_gbm_clean$top25_dom)
data_gbm_clean$bottom25_dom <- as.factor(data_gbm_clean$bottom25_dom)
data_gbm_clean$infections_period <- as.numeric(data_gbm_clean$infections_accum > 1000)
data_gbm_clean$infections_period <- as.factor(data_gbm_clean$infections_period)
str(data_gbm_clean)tibble [24,412 × 49] (S3: tbl_df/tbl/data.frame)
$ mls_number : chr [1:24412] "CNNN5274" "CNNN5241" "CNN104918" "CNN104870" ...
$ property_type : Factor w/ 6 levels "CND","DUP","OTH",..: 5 5 5 5 5 5 5 5 5 5 ...
$ ac_type : Factor w/ 3 levels "central","none",..: 1 3 1 1 1 1 1 1 1 1 ...
$ list_price : num [1:24412] 187000 250000 224900 225000 274900 ...
$ patio : Factor w/ 2 levels "0","1": 1 1 1 2 2 1 2 2 2 2 ...
$ school_general : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ photo_count : num [1:24412] 0 0 0 0 25 2 6 17 17 15 ...
$ pool : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 2 1 2 1 ...
$ roof_type : Factor w/ 4 levels "metal","other",..: 3 3 2 3 3 3 2 3 2 2 ...
$ gas_type : Factor w/ 5 levels "butane","natural",..: 5 5 5 5 5 5 5 5 5 5 ...
$ out_building : Factor w/ 2 levels "0","1": 1 1 2 1 1 1 1 1 2 1 ...
$ area_living : num [1:24412] 2054 2120 2078 1923 2184 ...
$ land_acres : num [1:24412] 0.28 0.4 0.29 0.36 0.82 0.36 1 1.27 0.63 2.01 ...
$ appliances : Factor w/ 2 levels "0","1": 2 1 2 2 2 2 2 2 2 2 ...
$ garage : Factor w/ 2 levels "0","1": 2 2 1 2 2 2 2 2 2 2 ...
$ property_condition : Factor w/ 3 levels "excellent","new",..: 3 3 3 3 3 3 3 3 3 3 ...
$ energy_efficient : Factor w/ 2 levels "0","1": 1 1 1 1 2 1 1 2 1 2 ...
$ exterior_type : Factor w/ 5 levels "brick","metal",..: 3 3 4 4 1 3 4 1 3 3 ...
$ exterior_features : Factor w/ 6 levels "balcony","courtyard",..: 4 4 4 5 5 3 4 5 3 3 ...
$ fireplace : Factor w/ 2 levels "0","1": 2 1 1 2 2 2 2 2 2 2 ...
$ foundation_type : Factor w/ 3 levels "raised","slab",..: 1 2 1 2 2 3 2 2 2 2 ...
$ area_total : num [1:24412] 2254 2120 2962 2550 3510 ...
$ beds_total : Factor w/ 7 levels "0","1","2","3",..: 4 5 4 4 4 4 5 4 5 5 ...
$ bath_full : Factor w/ 7 levels "0","1","2","3",..: 3 3 3 3 3 3 3 3 3 3 ...
$ bath_half : Factor w/ 6 levels "0","1","2","3",..: 1 1 1 2 1 1 1 1 2 1 ...
$ age : num [1:24412] 82 9 70 27 7 6 38 32 15 5 ...
$ dom : num [1:24412] 78 83 89 203 231 54 144 108 26 25 ...
$ sold_price : num [1:24412] 169000 245000 230000 220000 272000 ...
$ sold_date : num [1:24412] 16843 17123 17228 17336 17324 ...
$ sewer_type : Factor w/ 3 levels "city","septic",..: 1 1 1 2 2 1 3 3 1 3 ...
$ property_style : Factor w/ 2 levels "mobile","not_mobile": 2 2 2 2 2 2 2 2 2 2 ...
$ city_limits : Factor w/ 2 levels "0","1": 2 2 2 2 2 2 2 2 2 2 ...
$ subdivision : Factor w/ 2 levels "0","1": 2 1 2 2 2 2 2 2 2 2 ...
$ water_type : Factor w/ 2 levels "public","well": 1 1 1 1 1 1 1 1 1 1 ...
$ waterfront : Factor w/ 2 levels "0","1": 1 1 1 2 1 1 1 1 1 1 ...
$ infections_daily : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ infections_accum : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ corona_date_split : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ infections_3mma : num [1:24412] 0 0 0 0 0 0 0 0 0 0 ...
$ top25_sold_price : Factor w/ 2 levels "0","1": 1 2 1 1 2 2 1 2 2 2 ...
$ top50_sold_price : num [1:24412] 0 1 1 1 1 1 1 1 1 1 ...
$ bottom25_sold_price : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ top25_area_living : Factor w/ 2 levels "0","1": 2 2 2 1 2 2 2 2 2 1 ...
$ bottom25_area_living: Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ top25_age : Factor w/ 2 levels "0","1": 2 1 2 1 1 1 1 1 1 1 ...
$ bottom25_age : Factor w/ 2 levels "0","1": 1 2 1 1 2 2 1 1 2 2 ...
$ top25_dom : Factor w/ 2 levels "0","1": 1 1 1 2 2 1 2 1 1 1 ...
$ bottom25_dom : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...
$ infections_period : Factor w/ 2 levels "0","1": 1 1 1 1 1 1 1 1 1 1 ...# Remove this weird '20' level is bath_full
levels(data_gbm_clean$bath_full)[1] "0" "1" "2" "3" "4" "6" "20"is.na(data_gbm_clean$bath_full) <- data_gbm_clean$bath_full == "20"
data_gbm_clean$bath_full <- factor(data_gbm_clean$bath_full)
levels(data_gbm_clean$bath_full)[1] "0" "1" "2" "3" "4" "6"# Remove beds_total > 5
levels(data_gbm_clean$beds_total)[1] "0" "1" "2" "3" "4" "5" "6"is.na(data_gbm_clean$beds_total) <- data_gbm_clean$beds_total == "7"
data_gbm_clean$beds_total <- factor(data_gbm_clean$beds_total)
is.na(data_gbm_clean$beds_total) <- data_gbm_clean$beds_total == "6"
data_gbm_clean$beds_total <- factor(data_gbm_clean$beds_total)
# Non_linear Additions
data_gbm_clean$age_2 <- I(data_gbm_clean$age^2)
data_gbm_clean$area_living_2 <- I(data_gbm_clean$area_living^2)
# Removals
data_gbm_clean <- subset(data_gbm_clean, select = -c(area_total, list_price))
names(data_gbm_clean) [1] "mls_number" "property_type" "ac_type" "patio"
[5] "school_general" "photo_count" "pool" "roof_type"
[9] "gas_type" "out_building" "area_living" "land_acres"
[13] "appliances" "garage" "property_condition" "energy_efficient"
[17] "exterior_type" "exterior_features" "fireplace" "foundation_type"
[21] "beds_total" "bath_full" "bath_half" "age"
[25] "dom" "sold_price" "sold_date" "sewer_type"
[29] "property_style" "city_limits" "subdivision" "water_type"
[33] "waterfront" "infections_daily" "infections_accum" "corona_date_split"
[37] "infections_3mma" "top25_sold_price" "top50_sold_price" "bottom25_sold_price"
[41] "top25_area_living" "bottom25_area_living" "top25_age" "bottom25_age"
[45] "top25_dom" "bottom25_dom" "infections_period" "age_2"
[49] "area_living_2" # Remove other currently non-relevant variables
data_gbm_clean <- subset(data_gbm_clean, select = -c(mls_number, infections_accum, corona_date_split, top25_sold_price,
top50_sold_price, bottom25_sold_price,infections_period, infections_daily))
names(data_gbm_clean) [1] "property_type" "ac_type" "patio" "school_general"
[5] "photo_count" "pool" "roof_type" "gas_type"
[9] "out_building" "area_living" "land_acres" "appliances"
[13] "garage" "property_condition" "energy_efficient" "exterior_type"
[17] "exterior_features" "fireplace" "foundation_type" "beds_total"
[21] "bath_full" "bath_half" "age" "dom"
[25] "sold_price" "sold_date" "sewer_type" "property_style"
[29] "city_limits" "subdivision" "water_type" "waterfront"
[33] "infections_3mma" "top25_area_living" "bottom25_area_living" "top25_age"
[37] "bottom25_age" "top25_dom" "bottom25_dom" "age_2"
[41] "area_living_2" 1. Data Splitting
# Create training (70%) and test (30%) sets for data.
# Use set.seed for reproducibility
set.seed(1)
split <- initial_split(data_gbm_clean, prop = .7)
train <- training(split)
test <- testing(split)
names(train) [1] "property_type" "ac_type" "patio" "school_general"
[5] "photo_count" "pool" "roof_type" "gas_type"
[9] "out_building" "area_living" "land_acres" "appliances"
[13] "garage" "property_condition" "energy_efficient" "exterior_type"
[17] "exterior_features" "fireplace" "foundation_type" "beds_total"
[21] "bath_full" "bath_half" "age" "dom"
[25] "sold_price" "sold_date" "sewer_type" "property_style"
[29] "city_limits" "subdivision" "water_type" "waterfront"
[33] "infections_3mma" "top25_area_living" "bottom25_area_living" "top25_age"
[37] "bottom25_age" "top25_dom" "bottom25_dom" "age_2"
[41] "area_living_2" 2. Structure Data: One-Hot Encoding
# variable names
features <- setdiff(names(train), "sold_price")
features [1] "property_type" "ac_type" "patio" "school_general"
[5] "photo_count" "pool" "roof_type" "gas_type"
[9] "out_building" "area_living" "land_acres" "appliances"
[13] "garage" "property_condition" "energy_efficient" "exterior_type"
[17] "exterior_features" "fireplace" "foundation_type" "beds_total"
[21] "bath_full" "bath_half" "age" "dom"
[25] "sold_date" "sewer_type" "property_style" "city_limits"
[29] "subdivision" "water_type" "waterfront" "infections_3mma"
[33] "top25_area_living" "bottom25_area_living" "top25_age" "bottom25_age"
[37] "top25_dom" "bottom25_dom" "age_2" "area_living_2" # Create the treatment plan from the training data
treatplan <- vtreat::designTreatmentsZ(train, features, verbose = FALSE)
# Get the "clean" variable names from the scoreFrame
new_vars <- treatplan %>%
magrittr::use_series(scoreFrame) %>%
dplyr::filter(code %in% c("clean", "lev")) %>%
magrittr::use_series(varName)
# Prepare the training data
features_train <- vtreat::prepare(treatplan, train, varRestriction = new_vars) %>% as.matrix()
response_train <- train$sold_price
# Prepare the test data
features_test <- vtreat::prepare(treatplan, test, varRestriction = new_vars) %>% as.matrix()
response_test <- test$sold_price
# dimensions of one-hot encoded data
dim(features_train)[1] 17088 102dim(features_test)[1] 7324 1023. Base Model with Early Stoppage
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
# reproducibility
set.seed(1)
start_time <- Sys.time()
xgb.fit1 <- xgb.cv(
data = features_train,
label = response_train,
nrounds = 1000,
nfold = 5,
objective = "reg:linear", # for regression models
verbose = 0, # silent,
early_stopping_rounds = 10 # stop if no improvement for 10 consecutive trees
)[11:27:05] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.
[11:27:05] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.
[11:27:05] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.
[11:27:05] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.
[11:27:05] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.end_time <- Sys.time()
time_taken <- end_time - start_time
time_takenTime difference of 19.11896 secs# get number of trees that minimize error
xgb.fit1$evaluation_log %>%
dplyr::summarise(
ntrees.train = which(train_rmse_mean == min(train_rmse_mean))[1],
rmse.train = min(train_rmse_mean),
ntrees.test = which(test_rmse_mean == min(test_rmse_mean))[1],
rmse.test = min(test_rmse_mean),
)
# plot error vs number trees
ggplot(xgb.fit1$evaluation_log) +
geom_line(aes(iter, train_rmse_mean), color = "red") +
geom_line(aes(iter, test_rmse_mean), color = "blue")
stopCluster(cl)4. Hyperparameter Tuning
# create hyperparameter grid
hyper_grid <- expand.grid(
eta = c(.01, .05, .1, .3),
max_depth = c(1, 3, 5, 7),
min_child_weight = c(1, 3, 5, 7),
subsample = c(.65, .8, 1),
colsample_bytree = c(.8, .9, 1),
optimal_trees = 0, # a place to dump results
min_RMSE = 0 # a place to dump results
)
nrow(hyper_grid)
## [1] 576
# Manual Grid Search
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
for(i in 1:nrow(hyper_grid)) {
# create parameter list
params <- list(
eta = hyper_grid$eta[i],
max_depth = hyper_grid$max_depth[i],
min_child_weight = hyper_grid$min_child_weight[i],
subsample = hyper_grid$subsample[i],
colsample_bytree = hyper_grid$colsample_bytree[i]
)
# reproducibility
set.seed(007)
# train model
xgb.tune <- xgb.cv(
params = params,
data = features_train,
label = response_train,
nrounds = 5000,
nfold = 5,
objective = "reg:linear", # for regression models
verbose = 0, # silent,
early_stopping_rounds = 10 # stop if no improvement for 10 consecutive trees
)
# add min training error and trees to grid
hyper_grid$optimal_trees[i] <- which.min(xgb.tune$evaluation_log$test_rmse_mean)
hyper_grid$min_RMSE[i] <- min(xgb.tune$evaluation_log$test_rmse_mean)
}
hyper_grid %>%
dplyr::arrange(min_RMSE) %>%
head(10)
stopCluster(cl)
# Order: rank, eta, max_depth, min_child_weight, subsample, colsample_bytree,
# optimal_trees, min_RMSE
# 1 0.01 7 1 0.65 0.9 1827 41952.29
# 2 0.01 7 5 0.65 0.9 1747 41967.81
# 3 0.01 7 3 0.65 1.0 1695 42002.20
# 4 0.01 7 1 0.65 1.0 1670 42004.94
# 5 0.01 7 3 0.65 0.8 1583 42008.42
# 6 0.01 7 7 0.65 0.9 1607 42016.38
# 7 0.01 7 5 0.65 1.0 1556 42029.93
# 8 0.01 7 5 0.65 0.8 1479 42038.12
# 9 0.01 7 7 0.65 1.0 1555 42045.10
#10 0.01 7 7 0.65 0.8 1478 42065.54
4.1 Final Model with Tuned Hypers
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
start_time <- proc.time()
# parameter list
params <- list(
eta = 0.01,
max_depth = 7,
min_child_weight = 1,
subsample = 0.65,
colsample_bytree = 0.9
)
# train final model
xgb.fit.final <- xgboost(
params = params,
data = features_train,
label = response_train,
nrounds = 1827,
objective = "reg:linear",
verbose = 0
)[16:36:13] WARNING: amalgamation/../src/objective/regression_obj.cu:188: reg:linear is now deprecated in favor of reg:squarederror.end_time <- proc.time()
time_taken <- end_time - start_time
time_taken user system elapsed
168.883 1.649 173.325 stopCluster(cl)
# Test Error of final Model
# predict values for test data
pred <- predict(xgb.fit.final, features_test)
# results
caret::RMSE(pred, response_test)[1] 43140.75## [1] 21319.4
# plot error vs number trees
ggplot(xgb.fit.final$evaluation_log) +
geom_line(aes(iter, train_rmse), color = "red")
# plot error vs number trees
ggplot(xgb.fit.final$evaluation_log) +
geom_line(aes(iter, train_rmse), color = "red")
NA
NA5. Model Analysis and Visualizations
5.1 Variable Importance
vip::vip(xgb.fit.final)
# create importance matrix
importance_matrix <- xgb.importance(model = xgb.fit.final)
# variable importance plot
xgb.plot.importance(importance_matrix, top_n = 20, measure = "Gain")
5.2 Partial Dependence Plots
5.2.1 Corona
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "infections_3mma", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "infections_3mma", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")Warning: `fun.y` is deprecated. Use `fun` instead.
Warning: Ignoring unknown parameters: csidesgridExtra::grid.arrange(pdp, ice, nrow = 2)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat.id"]]` is discouraged. Use `.data[["yhat.id"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
stopCluster(cl)5.2.3 Living Area
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "area_living", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "area_living", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")Warning: `fun.y` is deprecated. Use `fun` instead.
Warning: Ignoring unknown parameters: csidesgridExtra::grid.arrange(pdp, ice, nrow = 2)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat.id"]]` is discouraged. Use `.data[["yhat.id"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
stopCluster(cl)5.2.4.1 Multi_Variable PDP
gridExtra::grid.arrange(pdp_heat_con, nrow = 1)
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
pdp_3d5.2.4 Days On Market
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "dom", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "dom", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")Warning: `fun.y` is deprecated. Use `fun` instead.
Warning: Ignoring unknown parameters: csidesgridExtra::grid.arrange(pdp, ice, nrow = 2)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat.id"]]` is discouraged. Use `.data[["yhat.id"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
stopCluster(cl)5.2.4.1 Multi_Variable PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
# Heatmap with Contour
pdp_heat_con <- xgb.fit.final %>%
partial(pred.var = c("infections_3mma", "dom"), chull = TRUE,
progress = "text", train = features_train, levelplot = FALSE) %>%
autoplot(rug = TRUE,
contour = TRUE,
contour.color = "#F0FFF0",
train = features_train,
main = "Infections and Days on Market",
xlab = "Daily Infections (3mma)",
ylab = "Days on Market",
legend.title = "Sold Price"
)
|
| | 0%
|
|= | 1%
|
|== | 1%
|
|== | 2%
|
|=== | 2%
|
|=== | 3%
|
|==== | 3%
|
|==== | 4%
|
|===== | 4%
|
|===== | 5%
|
|====== | 5%
|
|====== | 6%
|
|======= | 6%
|
|======= | 7%
|
|======== | 7%
|
|======== | 8%
|
|========= | 8%
|
|========= | 9%
|
|========== | 9%
|
|========== | 10%
|
|=========== | 10%
|
|=========== | 11%
|
|============ | 11%
|
|============ | 12%
|
|============= | 12%
|
|============= | 13%
|
|============== | 13%
|
|============== | 14%
|
|=============== | 14%
|
|=============== | 15%
|
|================ | 15%
|
|================ | 16%
|
|================= | 16%
|
|================== | 17%
|
|=================== | 17%
|
|=================== | 18%
|
|==================== | 18%
|
|==================== | 19%
|
|===================== | 19%
|
|===================== | 20%
|
|====================== | 20%
|
|====================== | 21%
|
|======================= | 21%
|
|======================= | 22%
|
|======================== | 22%
|
|======================== | 23%
|
|========================= | 23%
|
|========================= | 24%
|
|========================== | 24%
|
|========================== | 25%
|
|=========================== | 25%
|
|=========================== | 26%
|
|============================ | 26%
|
|============================ | 27%
|
|============================= | 27%
|
|============================= | 28%
|
|============================== | 28%
|
|============================== | 29%
|
|=============================== | 29%
|
|=============================== | 30%
|
|================================ | 30%
|
|================================ | 31%
|
|================================= | 31%
|
|================================= | 32%
|
|================================== | 32%
|
|================================== | 33%
|
|=================================== | 33%
|
|==================================== | 34%
|
|===================================== | 34%
|
|===================================== | 35%
|
|====================================== | 35%
|
|====================================== | 36%
|
|======================================= | 36%
|
|======================================= | 37%
|
|======================================== | 37%
|
|======================================== | 38%
|
|========================================= | 38%
|
|========================================= | 39%
|
|========================================== | 39%
|
|========================================== | 40%
|
|=========================================== | 40%
|
|=========================================== | 41%
|
|============================================ | 41%
|
|============================================ | 42%
|
|============================================= | 42%
|
|============================================= | 43%
|
|============================================== | 43%
|
|============================================== | 44%
|
|=============================================== | 44%
|
|=============================================== | 45%
|
|================================================ | 45%
|
|================================================ | 46%
|
|================================================= | 46%
|
|================================================= | 47%
|
|================================================== | 47%
|
|================================================== | 48%
|
|=================================================== | 48%
|
|=================================================== | 49%
|
|==================================================== | 49%
|
|===================================================== | 50%
|
|====================================================== | 51%
|
|======================================================= | 51%
|
|======================================================= | 52%
|
|======================================================== | 52%
|
|======================================================== | 53%
|
|========================================================= | 53%
|
|========================================================= | 54%
|
|========================================================== | 54%
|
|========================================================== | 55%
|
|=========================================================== | 55%
|
|=========================================================== | 56%
|
|============================================================ | 56%
|
|============================================================ | 57%
|
|============================================================= | 57%
|
|============================================================= | 58%
|
|============================================================== | 58%
|
|============================================================== | 59%
|
|=============================================================== | 59%
|
|=============================================================== | 60%
|
|================================================================ | 60%
|
|================================================================ | 61%
|
|================================================================= | 61%
|
|================================================================= | 62%
|
|================================================================== | 62%
|
|================================================================== | 63%
|
|=================================================================== | 63%
|
|=================================================================== | 64%
|
|==================================================================== | 64%
|
|==================================================================== | 65%
|
|===================================================================== | 65%
|
|===================================================================== | 66%
|
|====================================================================== | 66%
|
|======================================================================= | 67%
|
|======================================================================== | 67%
|
|======================================================================== | 68%
|
|========================================================================= | 68%
|
|========================================================================= | 69%
|
|========================================================================== | 69%
|
|========================================================================== | 70%
|
|=========================================================================== | 70%
|
|=========================================================================== | 71%
|
|============================================================================ | 71%
|
|============================================================================ | 72%
|
|============================================================================= | 72%
|
|============================================================================= | 73%
|
|============================================================================== | 73%
|
|============================================================================== | 74%
|
|=============================================================================== | 74%
|
|=============================================================================== | 75%
|
|================================================================================ | 75%
|
|================================================================================ | 76%
|
|================================================================================= | 76%
|
|================================================================================= | 77%
|
|================================================================================== | 77%
|
|================================================================================== | 78%
|
|=================================================================================== | 78%
|
|=================================================================================== | 79%
|
|==================================================================================== | 79%
|
|==================================================================================== | 80%
|
|===================================================================================== | 80%
|
|===================================================================================== | 81%
|
|====================================================================================== | 81%
|
|====================================================================================== | 82%
|
|======================================================================================= | 82%
|
|======================================================================================= | 83%
|
|======================================================================================== | 83%
|
|========================================================================================= | 84%
|
|========================================================================================== | 84%
|
|========================================================================================== | 85%
|
|=========================================================================================== | 85%
|
|=========================================================================================== | 86%
|
|============================================================================================ | 86%
|
|============================================================================================ | 87%
|
|============================================================================================= | 87%
|
|============================================================================================= | 88%
|
|============================================================================================== | 88%
|
|============================================================================================== | 89%
|
|=============================================================================================== | 89%
|
|=============================================================================================== | 90%
|
|================================================================================================ | 90%
|
|================================================================================================ | 91%
|
|================================================================================================= | 91%
|
|================================================================================================= | 92%
|
|================================================================================================== | 92%
|
|================================================================================================== | 93%
|
|=================================================================================================== | 93%
|
|=================================================================================================== | 94%
|
|==================================================================================================== | 94%
|
|==================================================================================================== | 95%
|
|===================================================================================================== | 95%
|
|===================================================================================================== | 96%
|
|====================================================================================================== | 96%
|
|====================================================================================================== | 97%
|
|======================================================================================================= | 97%
|
|======================================================================================================= | 98%
|
|======================================================================================================== | 98%
|
|======================================================================================================== | 99%
|
|========================================================================================================= | 99%
|
|==========================================================================================================| 100%# 3D Graphing
# Create 3D data matrix
infections_3d <- pdp_heat_con$data$infections_3mma
dom_3d <- pdp_heat_con$data$dom
yhat_3d <- pdp_heat_con$data$yhat
pdp_mat <- data.frame(infections_3d, dom_3d, yhat_3d) # Datafram for plotly 3D model
# Axis Titles
axx <- list(title = "Infections Daily")
axy <- list(title = "Days on Market")
axz <- list(title = "Price Sold")
# Colors: Manually matching plot standard gradient
very_low <- "#460f5c"
low <- "#2c728e"
med <- "#27ad81"
high <- "#f4e61e"
pdp_3d <- plot_ly(pdp_mat, x = ~infections_3d, y = ~dom_3d, z = ~yhat_3d,
type = 'mesh3d', intensity = ~yhat_3d,
colors = colorRamp(c(very_low, med, high)))
pdp_3d <- pdp_3d %>% layout(scene = list(xaxis=axx,yaxis=axy,zaxis=axz)) # Axis labs
pdp_3d <- hide_colorbar(pdp_3d) # Hide legend
# Print out
gridExtra::grid.arrange(pdp_heat_con, nrow = 1)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
pdp_3d
stopCluster(cl)5.2.5 Age
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "age", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "age", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")Warning: `fun.y` is deprecated. Use `fun` instead.
Warning: Ignoring unknown parameters: csidesgridExtra::grid.arrange(pdp, ice, nrow = 2)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat.id"]]` is discouraged. Use `.data[["yhat.id"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `x.rug[[1L]]` is discouraged. Use `.data[[1L]]` instead.
stopCluster(cl)5.2.4.1 Multi_Variable PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
# Heatmap with Contour
pdp_heat_con <- xgb.fit.final %>%
partial(pred.var = c("infections_3mma", "age"), chull = TRUE,
progress = "text", train = features_train, levelplot = FALSE) %>%
autoplot(rug = TRUE,
contour = TRUE,
contour.color = "#F0FFF0",
train = features_train,
main = "Infections and Age of Property",
xlab = "Daily Infections (3mma)",
ylab = "Age of Property",
legend.title = "Sold Price"
)
|
| | 0%
|
|= | 0%
|
|= | 1%
|
|== | 1%
|
|== | 2%
|
|=== | 2%
|
|=== | 3%
|
|==== | 3%
|
|==== | 4%
|
|===== | 4%
|
|===== | 5%
|
|====== | 5%
|
|====== | 6%
|
|======= | 6%
|
|======= | 7%
|
|======== | 7%
|
|======== | 8%
|
|========= | 8%
|
|========= | 9%
|
|========== | 9%
|
|========== | 10%
|
|=========== | 10%
|
|=========== | 11%
|
|============ | 11%
|
|============ | 12%
|
|============= | 12%
|
|============= | 13%
|
|============== | 13%
|
|============== | 14%
|
|=============== | 14%
|
|=============== | 15%
|
|================ | 15%
|
|================ | 16%Warning in .Internal(eval(expr, envir, enclos)) :
closing unused connection 12 (<-localhost:11516)
Warning in .Internal(eval(expr, envir, enclos)) :
closing unused connection 11 (<-localhost:11516)
Warning in .Internal(eval(expr, envir, enclos)) :
closing unused connection 10 (<-localhost:11516)
Warning in .Internal(eval(expr, envir, enclos)) :
closing unused connection 9 (<-localhost:11516)
Warning in .Internal(eval(expr, envir, enclos)) :
closing unused connection 8 (<-localhost:11516)
|
|================= | 16%
|
|================= | 17%
|
|================== | 17%
|
|================== | 18%
|
|=================== | 18%
|
|=================== | 19%
|
|==================== | 19%
|
|==================== | 20%
|
|===================== | 20%
|
|===================== | 21%
|
|====================== | 21%
|
|====================== | 22%
|
|======================= | 22%
|
|======================= | 23%
|
|======================== | 23%
|
|======================== | 24%
|
|========================= | 24%
|
|========================== | 25%
|
|=========================== | 26%
|
|============================ | 26%
|
|============================ | 27%
|
|============================= | 27%
|
|============================= | 28%
|
|============================== | 28%
|
|============================== | 29%
|
|=============================== | 29%
|
|=============================== | 30%
|
|================================ | 30%
|
|================================ | 31%
|
|================================= | 31%
|
|================================= | 32%
|
|================================== | 32%
|
|================================== | 33%
|
|=================================== | 33%
|
|=================================== | 34%
|
|==================================== | 34%
|
|==================================== | 35%
|
|===================================== | 35%
|
|===================================== | 36%
|
|====================================== | 36%
|
|====================================== | 37%
|
|======================================= | 37%
|
|======================================= | 38%
|
|======================================== | 38%
|
|======================================== | 39%
|
|========================================= | 39%
|
|========================================= | 40%
|
|========================================== | 40%
|
|========================================== | 41%
|
|=========================================== | 41%
|
|=========================================== | 42%
|
|============================================ | 42%
|
|============================================ | 43%
|
|============================================= | 43%
|
|============================================= | 44%
|
|============================================== | 44%
|
|============================================== | 45%
|
|=============================================== | 45%
|
|=============================================== | 46%
|
|================================================ | 46%
|
|================================================ | 47%
|
|================================================= | 47%
|
|================================================= | 48%
|
|================================================== | 48%
|
|================================================== | 49%
|
|=================================================== | 49%
|
|=================================================== | 50%
|
|==================================================== | 50%
|
|===================================================== | 50%
|
|===================================================== | 51%
|
|====================================================== | 51%
|
|====================================================== | 52%
|
|======================================================= | 52%
|
|======================================================= | 53%
|
|======================================================== | 53%
|
|======================================================== | 54%
|
|========================================================= | 54%
|
|========================================================= | 55%
|
|========================================================== | 55%
|
|========================================================== | 56%
|
|=========================================================== | 56%
|
|=========================================================== | 57%
|
|============================================================ | 57%
|
|============================================================ | 58%
|
|============================================================= | 58%
|
|============================================================= | 59%
|
|============================================================== | 59%
|
|============================================================== | 60%
|
|=============================================================== | 60%
|
|=============================================================== | 61%
|
|================================================================ | 61%
|
|================================================================ | 62%
|
|================================================================= | 62%
|
|================================================================= | 63%
|
|================================================================== | 63%
|
|================================================================== | 64%
|
|=================================================================== | 64%
|
|=================================================================== | 65%
|
|==================================================================== | 65%
|
|==================================================================== | 66%
|
|===================================================================== | 66%
|
|===================================================================== | 67%
|
|====================================================================== | 67%
|
|====================================================================== | 68%
|
|======================================================================= | 68%
|
|======================================================================= | 69%
|
|======================================================================== | 69%
|
|======================================================================== | 70%
|
|========================================================================= | 70%
|
|========================================================================= | 71%
|
|========================================================================== | 71%
|
|========================================================================== | 72%
|
|=========================================================================== | 72%
|
|=========================================================================== | 73%
|
|============================================================================ | 73%
|
|============================================================================ | 74%
|
|============================================================================= | 74%
|
|============================================================================== | 75%
|
|=============================================================================== | 76%
|
|================================================================================ | 76%
|
|================================================================================ | 77%
|
|================================================================================= | 77%
|
|================================================================================= | 78%
|
|================================================================================== | 78%
|
|================================================================================== | 79%
|
|=================================================================================== | 79%
|
|=================================================================================== | 80%
|
|==================================================================================== | 80%
|
|==================================================================================== | 81%
|
|===================================================================================== | 81%
|
|===================================================================================== | 82%
|
|====================================================================================== | 82%
|
|====================================================================================== | 83%
|
|======================================================================================= | 83%
|
|======================================================================================= | 84%
|
|======================================================================================== | 84%
|
|======================================================================================== | 85%
|
|========================================================================================= | 85%
|
|========================================================================================= | 86%
|
|========================================================================================== | 86%
|
|========================================================================================== | 87%
|
|=========================================================================================== | 87%
|
|=========================================================================================== | 88%
|
|============================================================================================ | 88%
|
|============================================================================================ | 89%
|
|============================================================================================= | 89%
|
|============================================================================================= | 90%
|
|============================================================================================== | 90%
|
|============================================================================================== | 91%
|
|=============================================================================================== | 91%
|
|=============================================================================================== | 92%
|
|================================================================================================ | 92%
|
|================================================================================================ | 93%
|
|================================================================================================= | 93%
|
|================================================================================================= | 94%
|
|================================================================================================== | 94%
|
|================================================================================================== | 95%
|
|=================================================================================================== | 95%
|
|=================================================================================================== | 96%
|
|==================================================================================================== | 96%
|
|==================================================================================================== | 97%
|
|===================================================================================================== | 97%
|
|===================================================================================================== | 98%
|
|====================================================================================================== | 98%
|
|====================================================================================================== | 99%
|
|======================================================================================================= | 99%
|
|======================================================================================================= | 100%
|
|========================================================================================================| 100%# 3D Graphing
# Create 3D data matrix
infections_3d <- pdp_heat_con$data$infections_3mma
age_3d <- pdp_heat_con$data$age
yhat_3d <- pdp_heat_con$data$yhat
pdp_mat <- data.frame(infections_3d, age_3d, yhat_3d) # Datafram for plotly 3D model
# Axis Titles
axx <- list(title = "Infections Daily")
axy <- list(title = "Age of Property")
axz <- list(title = "Price Sold")
# Colors: Manually matching plot standard gradient
very_low <- "#460f5c"
low <- "#2c728e"
med <- "#27ad81"
high <- "#f4e61e"
pdp_3d <- plot_ly(pdp_mat, x = ~infections_3d, y = ~age_3d, z = ~yhat_3d,
type = 'mesh3d', intensity = ~yhat_3d,
colors = colorRamp(c(very_low, med, high)))
pdp_3d <- pdp_3d %>% layout(scene = list(xaxis=axx,yaxis=axy,zaxis=axz)) # Axis labs
pdp_3d <- hide_colorbar(pdp_3d) # Hide legend
# Print out
gridExtra::grid.arrange(pdp_heat_con, nrow = 1)Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
pdp_3d
stopCluster(cl)5.2.6 Sold Date
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "sold_date", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "sold_date", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")
gridExtra::grid.arrange(pdp, ice, nrow = 2)
stopCluster(cl)5.2.4.1 Multi_Variable PDP
gridExtra::grid.arrange(pdp_heat_con, nrow = 1)
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[[1L]]` is discouraged. Use `.data[[1L]]` instead.
Warning: Use of `object[[2L]]` is discouraged. Use `.data[[2L]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
Warning: Use of `object[["yhat"]]` is discouraged. Use `.data[["yhat"]]` instead.
pdp_3d5.2.7 Photo Count
5.2.4.1 Standard PDP
# Use parallel computing to speed up processing time
cl <- makePSOCKcluster(5)
registerDoParallel(cl)
pdp <- xgb.fit.final %>%
partial(pred.var = "photo_count", n.trees = 1827, grid.resolution = 100, train = features_train) %>%
autoplot(rug = TRUE, train = features_train) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("PDP")
ice <- xgb.fit.final %>%
partial(pred.var = "photo_count", n.trees = 1827, grid.resolution = 100, train = features_train, ice = TRUE) %>%
autoplot(rug = TRUE, train = features_train, alpha = .1, center = TRUE) +
scale_y_continuous(labels = scales::dollar) +
ggtitle("ICE")
gridExtra::grid.arrange(pdp, ice, nrow = 2)
stopCluster(cl)5.2.2 Price
5.2.2 Number of Bedrooms
Playground
LIME
# one-hot encode the local observations to be assessed.
local_obs_onehot <- vtreat::prepare(treatplan, local_obs, varRestriction = new_vars)
# apply LIME
explainer <- lime(data.frame(features_train), xgb.fit.final)
explanation <- explain(local_obs_onehot, explainer, n_features = 5)
plot_features(explanation)Predicting impact of higher cases of corona
# predict values for test data
feature_test <- features_test[1:2,]
pred <- predict(xgb.fit.final, feature_test)
pred
?predict
# results
caret::RMSE(pred, response_test)
## [1] 21319.3cl <- makePSOCKcluster(5)
registerDoParallel(cl)
# reproducibility
set.seed(2)
start_time <- proc.time()
xgb.fit.final <- xgboost(
data = features_train,
label = response_train,
nrounds = 950,
nfold = 5,
objective = "reg:linear", # for regression models
verbose = 0, # silent,
early_stopping_rounds = 10 # stop if no improvement for 10 consecutive trees
)
end_time <- proc.time()
time_taken <- end_time - start_time
time_taken
stopCluster(cl)End of Code
LS0tCnRpdGxlOiAiSGVkb25pYyBQcmljaW5nIE1vZGVscyB3aXRoIE1hY2hpbmUgTGVhcm5pbmciCm91dHB1dDoKICBodG1sX25vdGVib29rOiBkZWZhdWx0CiAgcGRmX2RvY3VtZW50OiBkZWZhdWx0CmNvZGVfZm9sZGluZzogaGlkZQpBdXRob3I6IFNhd3llciBCZW5zb24KLS0tCgoKU3RhcnQgb2YgQ29kZQoKTG9hZCBwYWNrYWdlcwpgYGB7ciBpbmNsdWRlPUZBTFNFfQpsaWJyYXJ5KHJzYW1wbGUpICAgICAgIyBkYXRhIHNwbGl0dGluZwpsaWJyYXJ5KGdibSkgICAgICAgICAgIyBiYXNpYyBpbXBsZW1lbnRhdGlvbgpsaWJyYXJ5KHhnYm9vc3QpICAgICAgIyBhIGZhc3RlciBpbXBsZW1lbnRhdGlvbiBvZiBnYm0KbGlicmFyeShjYXJldCkgICAgICAgICMgYW4gYWdncmVnYXRvciBwYWNrYWdlIGZvciBwZXJmb3JtaW5nIG1hbnkgbWFjaGluZSBsZWFybmluZyBtb2RlbHMKbGlicmFyeShoMm8pICAgICAgICAgICMgYSBqYXZhLWJhc2VkIHBsYXRmb3JtCmxpYnJhcnkocGRwKSAgICAgICAgICAjIG1vZGVsIHZpc3VhbGl6YXRpb24KbGlicmFyeShnZ3Bsb3QyKSAgICAgICMgbW9kZWwgdmlzdWFsaXphdGlvbgpsaWJyYXJ5KGxpbWUpICAgICAgICAgIyBtb2RlbCB2aXN1YWxpemF0aW9uCmxpYnJhcnkoZG9QYXJhbGxlbCkgICAjIFBhcmFsbGVsIGNvbXB1dGluZyB0byByZWR1Y2UgbW9kZWwgcnVuIHRpbWUgCmxpYnJhcnkocGxvdGx5KSAgICAgICAjIEZvciAzRCBwbG90dGluZyBpbiBnZ3Bsb3QyCgpgYGAKClJlYWQgaW4gRGF0YQpgYGB7cn0KIyBSZWFkIGluIGRhdGEKbGlicmFyeShyZWFkeGwpCmRhdGFfZ2JtX2NsZWFuIDwtIHJlYWRfZXhjZWwoIi9Vc2Vycy9zYXd5ZXJiZW5zb24vRG9jdW1lbnRzL01hc3RlciBUaGVzaXMvVGhlc2lzX0dpdGh1Yi9Nb2RlbHMvRGF0YS9OZXcgRGF0YS8zLiBkYXRhX2ZhY3Rvcl9jbGVhbmVkLnhsc3giKQoKIyBTdHJ1Y3R1cmUgQ2hhbmdlCmRhdGFfZ2JtX2NsZWFuJHByb3BlcnR5X3R5cGUgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJHByb3BlcnR5X3R5cGUpCmRhdGFfZ2JtX2NsZWFuJGFjX3R5cGUgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGFjX3R5cGUpCmRhdGFfZ2JtX2NsZWFuJHBhdGlvIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRwYXRpbykKZGF0YV9nYm1fY2xlYW4kc2Nob29sX2dlbmVyYWwgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJHNjaG9vbF9nZW5lcmFsKQpkYXRhX2dibV9jbGVhbiRwb29sIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRwb29sKQpkYXRhX2dibV9jbGVhbiRyb29mX3R5cGUgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJHJvb2ZfdHlwZSkKZGF0YV9nYm1fY2xlYW4kZ2FzX3R5cGUgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGdhc190eXBlKQpkYXRhX2dibV9jbGVhbiRvdXRfYnVpbGRpbmcgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJG91dF9idWlsZGluZykKZGF0YV9nYm1fY2xlYW4kYXBwbGlhbmNlcyA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kYXBwbGlhbmNlcykKZGF0YV9nYm1fY2xlYW4kZ2FyYWdlIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRnYXJhZ2UpCmRhdGFfZ2JtX2NsZWFuJHByb3BlcnR5X2NvbmRpdGlvbiA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kcHJvcGVydHlfY29uZGl0aW9uKQpkYXRhX2dibV9jbGVhbiRlbmVyZ3lfZWZmaWNpZW50IDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRlbmVyZ3lfZWZmaWNpZW50KQpkYXRhX2dibV9jbGVhbiRleHRlcmlvcl90eXBlIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRleHRlcmlvcl90eXBlKQpkYXRhX2dibV9jbGVhbiRleHRlcmlvcl9mZWF0dXJlcyA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kZXh0ZXJpb3JfZmVhdHVyZXMpCmRhdGFfZ2JtX2NsZWFuJGZpcmVwbGFjZSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kZmlyZXBsYWNlKQpkYXRhX2dibV9jbGVhbiRmb3VuZGF0aW9uX3R5cGUgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGZvdW5kYXRpb25fdHlwZSkKZGF0YV9nYm1fY2xlYW4kYmVkc190b3RhbCA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kYmVkc190b3RhbCkKZGF0YV9nYm1fY2xlYW4kYmF0aF9mdWxsIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRiYXRoX2Z1bGwpCmRhdGFfZ2JtX2NsZWFuJGJhdGhfaGFsZiA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kYmF0aF9oYWxmKQpkYXRhX2dibV9jbGVhbiRzZXdlcl90eXBlIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRzZXdlcl90eXBlKQpkYXRhX2dibV9jbGVhbiRwcm9wZXJ0eV9zdHlsZSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kcHJvcGVydHlfc3R5bGUpCmRhdGFfZ2JtX2NsZWFuJHN1YmRpdmlzaW9uIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRzdWJkaXZpc2lvbikKZGF0YV9nYm1fY2xlYW4kd2F0ZXJfdHlwZSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kd2F0ZXJfdHlwZSkKZGF0YV9nYm1fY2xlYW4kd2F0ZXJmcm9udCA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kd2F0ZXJmcm9udCkKZGF0YV9nYm1fY2xlYW4kc29sZF9kYXRlIDwtIG9wZW54bHN4Ojpjb252ZXJ0VG9EYXRlKGRhdGFfZ2JtX2NsZWFuJHNvbGRfZGF0ZSkKZGF0YV9nYm1fY2xlYW4kc29sZF9kYXRlIDwtIGFzLm51bWVyaWMoZGF0YV9nYm1fY2xlYW4kc29sZF9kYXRlKQpzdHIoZGF0YV9nYm1fY2xlYW4pCgojIFNwbGl0cwpkYXRhX2dibV9jbGVhbiRjaXR5X2xpbWl0cyA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kY2l0eV9saW1pdHMpCmRhdGFfZ2JtX2NsZWFuJGNvcm9uYV9kYXRlX3NwbGl0IDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRjb3JvbmFfZGF0ZV9zcGxpdCkKZGF0YV9nYm1fY2xlYW4kdG9wMjVfc29sZF9wcmljZSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kdG9wMjVfc29sZF9wcmljZSkKZGF0YV9nYm1fY2xlYW4kYm90dG9tMjVfc29sZF9wcmljZSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kYm90dG9tMjVfc29sZF9wcmljZSkKZGF0YV9nYm1fY2xlYW4kdG9wMjVfYXJlYV9saXZpbmcgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJHRvcDI1X2FyZWFfbGl2aW5nKQpkYXRhX2dibV9jbGVhbiRib3R0b20yNV9hcmVhX2xpdmluZyAgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGJvdHRvbTI1X2FyZWFfbGl2aW5nKQpkYXRhX2dibV9jbGVhbiR0b3AyNV9hZ2UgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJHRvcDI1X2FnZSkKZGF0YV9nYm1fY2xlYW4kYm90dG9tMjVfYWdlIDwtIGFzLmZhY3RvcihkYXRhX2dibV9jbGVhbiRib3R0b20yNV9hZ2UpCmRhdGFfZ2JtX2NsZWFuJHRvcDI1X2RvbSA8LSBhcy5mYWN0b3IoZGF0YV9nYm1fY2xlYW4kdG9wMjVfZG9tKQpkYXRhX2dibV9jbGVhbiRib3R0b20yNV9kb20gPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGJvdHRvbTI1X2RvbSkKZGF0YV9nYm1fY2xlYW4kaW5mZWN0aW9uc19wZXJpb2QgPC0gYXMubnVtZXJpYyhkYXRhX2dibV9jbGVhbiRpbmZlY3Rpb25zX2FjY3VtID4gMTAwMCkKZGF0YV9nYm1fY2xlYW4kaW5mZWN0aW9uc19wZXJpb2QgPC0gYXMuZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGluZmVjdGlvbnNfcGVyaW9kKQpzdHIoZGF0YV9nYm1fY2xlYW4pCgojIFJlbW92ZSB0aGlzIHdlaXJkICcyMCcgbGV2ZWwgaXMgYmF0aF9mdWxsCmxldmVscyhkYXRhX2dibV9jbGVhbiRiYXRoX2Z1bGwpCmlzLm5hKGRhdGFfZ2JtX2NsZWFuJGJhdGhfZnVsbCkgPC0gZGF0YV9nYm1fY2xlYW4kYmF0aF9mdWxsID09ICIyMCIKZGF0YV9nYm1fY2xlYW4kYmF0aF9mdWxsIDwtIGZhY3RvcihkYXRhX2dibV9jbGVhbiRiYXRoX2Z1bGwpCmxldmVscyhkYXRhX2dibV9jbGVhbiRiYXRoX2Z1bGwpCgojIFJlbW92ZSBiZWRzX3RvdGFsID4gNQpsZXZlbHMoZGF0YV9nYm1fY2xlYW4kYmVkc190b3RhbCkKaXMubmEoZGF0YV9nYm1fY2xlYW4kYmVkc190b3RhbCkgPC0gZGF0YV9nYm1fY2xlYW4kYmVkc190b3RhbCA9PSAiNyIgCmRhdGFfZ2JtX2NsZWFuJGJlZHNfdG90YWwgPC0gZmFjdG9yKGRhdGFfZ2JtX2NsZWFuJGJlZHNfdG90YWwpCmlzLm5hKGRhdGFfZ2JtX2NsZWFuJGJlZHNfdG90YWwpIDwtIGRhdGFfZ2JtX2NsZWFuJGJlZHNfdG90YWwgPT0gIjYiIApkYXRhX2dibV9jbGVhbiRiZWRzX3RvdGFsIDwtIGZhY3RvcihkYXRhX2dibV9jbGVhbiRiZWRzX3RvdGFsKQoKIyBOb25fbGluZWFyIEFkZGl0aW9ucwpkYXRhX2dibV9jbGVhbiRhZ2VfMiA8LSBJKGRhdGFfZ2JtX2NsZWFuJGFnZV4yKQpkYXRhX2dibV9jbGVhbiRhcmVhX2xpdmluZ18yIDwtIEkoZGF0YV9nYm1fY2xlYW4kYXJlYV9saXZpbmdeMikKCiMgUmVtb3ZhbHMKZGF0YV9nYm1fY2xlYW4gPC0gc3Vic2V0KGRhdGFfZ2JtX2NsZWFuLCBzZWxlY3QgPSAtYyhhcmVhX3RvdGFsLCBsaXN0X3ByaWNlKSkKbmFtZXMoZGF0YV9nYm1fY2xlYW4pCgojIFJlbW92ZSBvdGhlciBjdXJyZW50bHkgbm9uLXJlbGV2YW50IHZhcmlhYmxlcyAKZGF0YV9nYm1fY2xlYW4gPC0gc3Vic2V0KGRhdGFfZ2JtX2NsZWFuLCBzZWxlY3QgPSAtYyhtbHNfbnVtYmVyLCBpbmZlY3Rpb25zX2FjY3VtLCBjb3JvbmFfZGF0ZV9zcGxpdCwgdG9wMjVfc29sZF9wcmljZSwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG9wNTBfc29sZF9wcmljZSwgYm90dG9tMjVfc29sZF9wcmljZSxpbmZlY3Rpb25zX3BlcmlvZCwgaW5mZWN0aW9uc19kYWlseSkpCm5hbWVzKGRhdGFfZ2JtX2NsZWFuKQoKYGBgCgo8YnI+CgojIyMgMS4gRGF0YSBTcGxpdHRpbmcKYGBge3J9CiMgQ3JlYXRlIHRyYWluaW5nICg3MCUpIGFuZCB0ZXN0ICgzMCUpIHNldHMgZm9yIGRhdGEuCiMgVXNlIHNldC5zZWVkIGZvciByZXByb2R1Y2liaWxpdHkKc2V0LnNlZWQoMSkKc3BsaXQgPC0gaW5pdGlhbF9zcGxpdChkYXRhX2dibV9jbGVhbiwgcHJvcCA9IC43KQp0cmFpbiA8LSB0cmFpbmluZyhzcGxpdCkKdGVzdCAgPC0gdGVzdGluZyhzcGxpdCkKCm5hbWVzKHRyYWluKQpgYGAKCjxicj4KCiMjIyAyLiBTdHJ1Y3R1cmUgRGF0YTogT25lLUhvdCBFbmNvZGluZwpgYGB7cn0KIyB2YXJpYWJsZSBuYW1lcwpmZWF0dXJlcyA8LSBzZXRkaWZmKG5hbWVzKHRyYWluKSwgInNvbGRfcHJpY2UiKQpmZWF0dXJlcwoKIyBDcmVhdGUgdGhlIHRyZWF0bWVudCBwbGFuIGZyb20gdGhlIHRyYWluaW5nIGRhdGEKdHJlYXRwbGFuIDwtIHZ0cmVhdDo6ZGVzaWduVHJlYXRtZW50c1oodHJhaW4sIGZlYXR1cmVzLCB2ZXJib3NlID0gRkFMU0UpCgojIEdldCB0aGUgImNsZWFuIiB2YXJpYWJsZSBuYW1lcyBmcm9tIHRoZSBzY29yZUZyYW1lCm5ld192YXJzIDwtIHRyZWF0cGxhbiAlPiUKICBtYWdyaXR0cjo6dXNlX3NlcmllcyhzY29yZUZyYW1lKSAlPiUgICAgICAgIAogIGRwbHlyOjpmaWx0ZXIoY29kZSAlaW4lIGMoImNsZWFuIiwgImxldiIpKSAlPiUgCiAgbWFncml0dHI6OnVzZV9zZXJpZXModmFyTmFtZSkgICAgIAoKIyBQcmVwYXJlIHRoZSB0cmFpbmluZyBkYXRhCmZlYXR1cmVzX3RyYWluIDwtIHZ0cmVhdDo6cHJlcGFyZSh0cmVhdHBsYW4sIHRyYWluLCB2YXJSZXN0cmljdGlvbiA9IG5ld192YXJzKSAlPiUgYXMubWF0cml4KCkKcmVzcG9uc2VfdHJhaW4gPC0gdHJhaW4kc29sZF9wcmljZQoKIyBQcmVwYXJlIHRoZSB0ZXN0IGRhdGEKZmVhdHVyZXNfdGVzdCA8LSB2dHJlYXQ6OnByZXBhcmUodHJlYXRwbGFuLCB0ZXN0LCB2YXJSZXN0cmljdGlvbiA9IG5ld192YXJzKSAlPiUgYXMubWF0cml4KCkKcmVzcG9uc2VfdGVzdCA8LSB0ZXN0JHNvbGRfcHJpY2UKCiMgZGltZW5zaW9ucyBvZiBvbmUtaG90IGVuY29kZWQgZGF0YQpkaW0oZmVhdHVyZXNfdHJhaW4pCmRpbShmZWF0dXJlc190ZXN0KQpgYGAKCjxicj4KCiMjIyAzLiBCYXNlIE1vZGVsIHdpdGggRWFybHkgU3RvcHBhZ2UKYGBge3J9CiMgVXNlIHBhcmFsbGVsIGNvbXB1dGluZyB0byBzcGVlZCB1cCBwcm9jZXNzaW5nIHRpbWUKY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgojIHJlcHJvZHVjaWJpbGl0eQpzZXQuc2VlZCgxKQoKc3RhcnRfdGltZSA8LSBTeXMudGltZSgpCnhnYi5maXQxIDwtIHhnYi5jdigKICBkYXRhID0gZmVhdHVyZXNfdHJhaW4sCiAgbGFiZWwgPSByZXNwb25zZV90cmFpbiwKICBucm91bmRzID0gMTAwMCwKICBuZm9sZCA9IDUsCiAgb2JqZWN0aXZlID0gInJlZzpsaW5lYXIiLCAgIyBmb3IgcmVncmVzc2lvbiBtb2RlbHMKICB2ZXJib3NlID0gMCwgICAgICAgICAgICAgICAjIHNpbGVudCwKICBlYXJseV9zdG9wcGluZ19yb3VuZHMgPSAxMCAjIHN0b3AgaWYgbm8gaW1wcm92ZW1lbnQgZm9yIDEwIGNvbnNlY3V0aXZlIHRyZWVzCikKZW5kX3RpbWUgPC0gU3lzLnRpbWUoKQp0aW1lX3Rha2VuIDwtIGVuZF90aW1lIC0gc3RhcnRfdGltZQp0aW1lX3Rha2VuCgojIGdldCBudW1iZXIgb2YgdHJlZXMgdGhhdCBtaW5pbWl6ZSBlcnJvcgp4Z2IuZml0MSRldmFsdWF0aW9uX2xvZyAlPiUKICBkcGx5cjo6c3VtbWFyaXNlKAogICAgbnRyZWVzLnRyYWluID0gd2hpY2godHJhaW5fcm1zZV9tZWFuID09IG1pbih0cmFpbl9ybXNlX21lYW4pKVsxXSwKICAgIHJtc2UudHJhaW4gICA9IG1pbih0cmFpbl9ybXNlX21lYW4pLAogICAgbnRyZWVzLnRlc3QgID0gd2hpY2godGVzdF9ybXNlX21lYW4gPT0gbWluKHRlc3Rfcm1zZV9tZWFuKSlbMV0sCiAgICBybXNlLnRlc3QgICA9IG1pbih0ZXN0X3Jtc2VfbWVhbiksCiAgKQoKIyBwbG90IGVycm9yIHZzIG51bWJlciB0cmVlcwpnZ3Bsb3QoeGdiLmZpdDEkZXZhbHVhdGlvbl9sb2cpICsKICBnZW9tX2xpbmUoYWVzKGl0ZXIsIHRyYWluX3Jtc2VfbWVhbiksIGNvbG9yID0gInJlZCIpICsKICBnZW9tX2xpbmUoYWVzKGl0ZXIsIHRlc3Rfcm1zZV9tZWFuKSwgY29sb3IgPSAiYmx1ZSIpCgpzdG9wQ2x1c3RlcihjbCkKYGBgCgo8YnI+CgojIyMgNC4gSHlwZXJwYXJhbWV0ZXIgVHVuaW5nCmBgYHtyLCB3YXJuaW5nPUZBTFNFLCBhdHRyLm91dHB1dD0nc3R5bGU9Im1heC1oZWlnaHQ6IDI1MHB4OyInfQojIGNyZWF0ZSBoeXBlcnBhcmFtZXRlciBncmlkCmh5cGVyX2dyaWQgPC0gZXhwYW5kLmdyaWQoCiAgZXRhID0gYyguMDEsIC4wNSwgLjEsIC4zKSwKICBtYXhfZGVwdGggPSBjKDEsIDMsIDUsIDcpLAogIG1pbl9jaGlsZF93ZWlnaHQgPSBjKDEsIDMsIDUsIDcpLAogIHN1YnNhbXBsZSA9IGMoLjY1LCAuOCwgMSksIAogIGNvbHNhbXBsZV9ieXRyZWUgPSBjKC44LCAuOSwgMSksCiAgb3B0aW1hbF90cmVlcyA9IDAsICAgICAgICAgICAgICAgIyBhIHBsYWNlIHRvIGR1bXAgcmVzdWx0cwogIG1pbl9STVNFID0gMCAgICAgICAgICAgICAgICAgICAgICMgYSBwbGFjZSB0byBkdW1wIHJlc3VsdHMKKQoKbnJvdyhoeXBlcl9ncmlkKQojIyBbMV0gNTc2CgojIE1hbnVhbCBHcmlkIFNlYXJjaAojIFVzZSBwYXJhbGxlbCBjb21wdXRpbmcgdG8gc3BlZWQgdXAgcHJvY2Vzc2luZyB0aW1lCmNsIDwtIG1ha2VQU09DS2NsdXN0ZXIoNSkKcmVnaXN0ZXJEb1BhcmFsbGVsKGNsKQoKZm9yKGkgaW4gMTpucm93KGh5cGVyX2dyaWQpKSB7CiAgCiAgIyBjcmVhdGUgcGFyYW1ldGVyIGxpc3QKICBwYXJhbXMgPC0gbGlzdCgKICAgIGV0YSA9IGh5cGVyX2dyaWQkZXRhW2ldLAogICAgbWF4X2RlcHRoID0gaHlwZXJfZ3JpZCRtYXhfZGVwdGhbaV0sCiAgICBtaW5fY2hpbGRfd2VpZ2h0ID0gaHlwZXJfZ3JpZCRtaW5fY2hpbGRfd2VpZ2h0W2ldLAogICAgc3Vic2FtcGxlID0gaHlwZXJfZ3JpZCRzdWJzYW1wbGVbaV0sCiAgICBjb2xzYW1wbGVfYnl0cmVlID0gaHlwZXJfZ3JpZCRjb2xzYW1wbGVfYnl0cmVlW2ldCiAgKQogIAogICMgcmVwcm9kdWNpYmlsaXR5CiAgc2V0LnNlZWQoMDA3KQogIAogICMgdHJhaW4gbW9kZWwKICB4Z2IudHVuZSA8LSB4Z2IuY3YoCiAgICBwYXJhbXMgPSBwYXJhbXMsCiAgICBkYXRhID0gZmVhdHVyZXNfdHJhaW4sCiAgICBsYWJlbCA9IHJlc3BvbnNlX3RyYWluLAogICAgbnJvdW5kcyA9IDUwMDAsCiAgICBuZm9sZCA9IDUsCiAgICBvYmplY3RpdmUgPSAicmVnOmxpbmVhciIsICAjIGZvciByZWdyZXNzaW9uIG1vZGVscwogICAgdmVyYm9zZSA9IDAsICAgICAgICAgICAgICAgIyBzaWxlbnQsCiAgICBlYXJseV9zdG9wcGluZ19yb3VuZHMgPSAxMCAjIHN0b3AgaWYgbm8gaW1wcm92ZW1lbnQgZm9yIDEwIGNvbnNlY3V0aXZlIHRyZWVzCiAgKQogIAogICMgYWRkIG1pbiB0cmFpbmluZyBlcnJvciBhbmQgdHJlZXMgdG8gZ3JpZAogIGh5cGVyX2dyaWQkb3B0aW1hbF90cmVlc1tpXSA8LSB3aGljaC5taW4oeGdiLnR1bmUkZXZhbHVhdGlvbl9sb2ckdGVzdF9ybXNlX21lYW4pCiAgaHlwZXJfZ3JpZCRtaW5fUk1TRVtpXSA8LSBtaW4oeGdiLnR1bmUkZXZhbHVhdGlvbl9sb2ckdGVzdF9ybXNlX21lYW4pCn0KCmh5cGVyX2dyaWQgJT4lCiAgZHBseXI6OmFycmFuZ2UobWluX1JNU0UpICU+JQogIGhlYWQoMTApCgoKc3RvcENsdXN0ZXIoY2wpCgojIE9yZGVyOiAgIHJhbmssIGV0YSwgbWF4X2RlcHRoLCBtaW5fY2hpbGRfd2VpZ2h0LCBzdWJzYW1wbGUsIGNvbHNhbXBsZV9ieXRyZWUsCiMgICAgICAgICAgb3B0aW1hbF90cmVlcywgbWluX1JNU0UgIAoKIyAxCTAuMDEJNwkgICAgICAgICAxCTAuNjUJMC45CTE4MjcJNDE5NTIuMjkKIyAyCTAuMDEJNwkgICAgICAgICA1CTAuNjUJMC45CTE3NDcJNDE5NjcuODEKIyAzCTAuMDEJNwkgICAgICAgICAzCTAuNjUJMS4wCTE2OTUJNDIwMDIuMjAKIyA0CTAuMDEJNwkgICAgICAgICAxCTAuNjUJMS4wCTE2NzAJNDIwMDQuOTQKIyA1CTAuMDEJNwkgICAgICAgICAzCTAuNjUJMC44CTE1ODMJNDIwMDguNDIKIyA2CTAuMDEJNwkgICAgICAgICA3CTAuNjUJMC45CTE2MDcJNDIwMTYuMzgKIyA3CTAuMDEJNwkgICAgICAgICA1CTAuNjUJMS4wCTE1NTYJNDIwMjkuOTMKIyA4CTAuMDEJNwkgICAgICAgICA1CTAuNjUJMC44CTE0NzkJNDIwMzguMTIKIyA5CTAuMDEJNwkgICAgICAgICA3CTAuNjUJMS4wCTE1NTUJNDIwNDUuMTAKIzEwCTAuMDEJNwkgICAgICAgICA3CTAuNjUJMC44CTE0NzgJNDIwNjUuNTQKCgoKYGBgCgoKIyMjIDQuMSBGaW5hbCBNb2RlbCB3aXRoIFR1bmVkIEh5cGVycwpgYGB7cn0KCiMgVXNlIHBhcmFsbGVsIGNvbXB1dGluZyB0byBzcGVlZCB1cCBwcm9jZXNzaW5nIHRpbWUKY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgpzdGFydF90aW1lIDwtIHByb2MudGltZSgpCiMgcGFyYW1ldGVyIGxpc3QKcGFyYW1zIDwtIGxpc3QoCiAgZXRhID0gMC4wMSwKICBtYXhfZGVwdGggPSA3LAogIG1pbl9jaGlsZF93ZWlnaHQgPSAxLAogIHN1YnNhbXBsZSA9IDAuNjUsCiAgY29sc2FtcGxlX2J5dHJlZSA9IDAuOQopCgojIHRyYWluIGZpbmFsIG1vZGVsCnhnYi5maXQuZmluYWwgPC0geGdib29zdCgKICBwYXJhbXMgPSBwYXJhbXMsCiAgZGF0YSA9IGZlYXR1cmVzX3RyYWluLAogIGxhYmVsID0gcmVzcG9uc2VfdHJhaW4sCiAgbnJvdW5kcyA9IDE4MjcsCiAgb2JqZWN0aXZlID0gInJlZzpsaW5lYXIiLAogIHZlcmJvc2UgPSAwCikKZW5kX3RpbWUgPC0gcHJvYy50aW1lKCkKdGltZV90YWtlbiA8LSBlbmRfdGltZSAtIHN0YXJ0X3RpbWUKdGltZV90YWtlbgoKc3RvcENsdXN0ZXIoY2wpCgoKIyBUZXN0IEVycm9yIG9mIGZpbmFsIE1vZGVsCiMgcHJlZGljdCB2YWx1ZXMgZm9yIHRlc3QgZGF0YQpwcmVkIDwtIHByZWRpY3QoeGdiLmZpdC5maW5hbCwgZmVhdHVyZXNfdGVzdCkKCiMgcmVzdWx0cwpjYXJldDo6Uk1TRShwcmVkLCByZXNwb25zZV90ZXN0KQojIyBbMV0gMjEzMTkuNAoKIyBwbG90IGVycm9yIHZzIG51bWJlciB0cmVlcwpnZ3Bsb3QoeGdiLmZpdC5maW5hbCRldmFsdWF0aW9uX2xvZykgKwogIGdlb21fbGluZShhZXMoaXRlciwgdHJhaW5fcm1zZSksIGNvbG9yID0gInJlZCIpCgojIHBsb3QgZXJyb3IgdnMgbnVtYmVyIHRyZWVzCmdncGxvdCh4Z2IuZml0LmZpbmFsJGV2YWx1YXRpb25fbG9nKSArCiAgZ2VvbV9saW5lKGFlcyhpdGVyLCB0cmFpbl9ybXNlKSwgY29sb3IgPSAicmVkIikKCgpgYGAKCjxicj4KCiMjIyA1LiBNb2RlbCBBbmFseXNpcyBhbmQgVmlzdWFsaXphdGlvbnMKCiMjIyMgNS4xIFZhcmlhYmxlIEltcG9ydGFuY2UKYGBge3J9CnZpcDo6dmlwKHhnYi5maXQuZmluYWwpCgojIGNyZWF0ZSBpbXBvcnRhbmNlIG1hdHJpeAppbXBvcnRhbmNlX21hdHJpeCA8LSB4Z2IuaW1wb3J0YW5jZShtb2RlbCA9IHhnYi5maXQuZmluYWwpCgojIHZhcmlhYmxlIGltcG9ydGFuY2UgcGxvdAp4Z2IucGxvdC5pbXBvcnRhbmNlKGltcG9ydGFuY2VfbWF0cml4LCB0b3BfbiA9IDIwLCBtZWFzdXJlID0gIkdhaW4iKQpgYGAKCiMjIyMgNS4yIFBhcnRpYWwgRGVwZW5kZW5jZSBQbG90cwojIyMjIyA1LjIuMSBDb3JvbmEKIyMjIyMjIDUuMi40LjEgU3RhbmRhcmQgUERQCmBgYHtyfQojIFVzZSBwYXJhbGxlbCBjb21wdXRpbmcgdG8gc3BlZWQgdXAgcHJvY2Vzc2luZyB0aW1lCmNsIDwtIG1ha2VQU09DS2NsdXN0ZXIoNSkKcmVnaXN0ZXJEb1BhcmFsbGVsKGNsKQoKcGRwIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9ICJpbmZlY3Rpb25zXzNtbWEiLCBuLnRyZWVzID0gMTgyNywgZ3JpZC5yZXNvbHV0aW9uID0gMTAwLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluKSAlPiUKICBhdXRvcGxvdChydWcgPSBUUlVFLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6ZG9sbGFyKSArCiAgZ2d0aXRsZSgiUERQIikKCmljZSA8LSB4Z2IuZml0LmZpbmFsICU+JQogIHBhcnRpYWwocHJlZC52YXIgPSAiaW5mZWN0aW9uc18zbW1hIiwgbi50cmVlcyA9IDE4MjcsIGdyaWQucmVzb2x1dGlvbiA9IDEwMCwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgaWNlID0gVFJVRSkgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgYWxwaGEgPSAuMSwgY2VudGVyID0gVFJVRSkgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmRvbGxhcikgKwogIGdndGl0bGUoIklDRSIpCgpncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHAsIGljZSwgbnJvdyA9IDIpCgpzdG9wQ2x1c3RlcihjbCkKYGBgCgoKIyMjIyMgNS4yLjMgTGl2aW5nIEFyZWEKIyMjIyMjIDUuMi40LjEgU3RhbmRhcmQgUERQCmBgYHtyfQojIFVzZSBwYXJhbGxlbCBjb21wdXRpbmcgdG8gc3BlZWQgdXAgcHJvY2Vzc2luZyB0aW1lCmNsIDwtIG1ha2VQU09DS2NsdXN0ZXIoNSkKcmVnaXN0ZXJEb1BhcmFsbGVsKGNsKQoKcGRwIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9ICJhcmVhX2xpdmluZyIsIG4udHJlZXMgPSAxODI3LCBncmlkLnJlc29sdXRpb24gPSAxMDAsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4pICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4pICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpkb2xsYXIpICsKICBnZ3RpdGxlKCJQRFAiKQoKaWNlIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9ICJhcmVhX2xpdmluZyIsIG4udHJlZXMgPSAxODI3LCBncmlkLnJlc29sdXRpb24gPSAxMDAsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sIGljZSA9IFRSVUUpICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sIGFscGhhID0gLjEsIGNlbnRlciA9IFRSVUUpICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpkb2xsYXIpICsKICBnZ3RpdGxlKCJJQ0UiKQoKZ3JpZEV4dHJhOjpncmlkLmFycmFuZ2UocGRwLCBpY2UsIG5yb3cgPSAyKQoKc3RvcENsdXN0ZXIoY2wpCmBgYAoKIyMjIyMjIDUuMi40LjEgTXVsdGlfVmFyaWFibGUgUERQCmBgYHtyfQojIFVzZSBwYXJhbGxlbCBjb21wdXRpbmcgdG8gc3BlZWQgdXAgcHJvY2Vzc2luZyB0aW1lCmNsIDwtIG1ha2VQU09DS2NsdXN0ZXIoNSkKcmVnaXN0ZXJEb1BhcmFsbGVsKGNsKQoKIyBIZWF0bWFwIHdpdGggQ29udG91cgpwZHBfaGVhdF9jb24gPC0geGdiLmZpdC5maW5hbCAlPiUKICBwYXJ0aWFsKHByZWQudmFyID0gYygiaW5mZWN0aW9uc18zbW1hIiwgImFyZWFfbGl2aW5nIiksIGNodWxsID0gVFJVRSwgCiAgICAgICAgICBwcm9ncmVzcyA9ICJ0ZXh0IiwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgbGV2ZWxwbG90ID0gRkFMU0UpICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIAogICAgICAgICAgIGNvbnRvdXIgPSBUUlVFLCAKICAgICAgICAgICBjb250b3VyLmNvbG9yID0gIiNGMEZGRjAiLCAKICAgICAgICAgICB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLAogICAgICAgICAgIG1haW4gPSAiSW5mZWN0aW9ucyBhbmQgTGl2aW5nIEFyZWEiLAogICAgICAgICAgIHhsYWIgPSAiRGFpbHkgSW5mZWN0aW9ucyAoM21tYSkiLAogICAgICAgICAgIHlsYWIgPSAiTGl2aW5nIEFyZWEiLAogICAgICAgICAgIGxlZ2VuZC50aXRsZSA9ICJTb2xkIFByaWNlIgogICAgICAgICAgICkKCiMgM0QgR3JhcGhpbmcKIyBDcmVhdGUgM0QgZGF0YSBtYXRyaXggCmluZmVjdGlvbnNfM2QgPC0gcGRwX2hlYXRfY29uJGRhdGEkaW5mZWN0aW9uc18zbW1hCmFyZWFfbGl2aW5nXzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJGFyZWFfbGl2aW5nCnloYXRfM2QgPC0gcGRwX2hlYXRfY29uJGRhdGEkeWhhdApwZHBfbWF0IDwtIGRhdGEuZnJhbWUoaW5mZWN0aW9uc18zZCwgYXJlYV9saXZpbmdfM2QsIHloYXRfM2QpICMgRGF0YWZyYW0gZm9yIHBsb3RseSAzRCBtb2RlbAoKIyBBeGlzIFRpdGxlcwpheHggPC0gbGlzdCh0aXRsZSA9ICJJbmZlY3Rpb25zIERhaWx5IikKYXh5IDwtIGxpc3QodGl0bGUgPSAiTGl2aW5nIEFyZWEiKQpheHogPC0gbGlzdCh0aXRsZSA9ICJQcmljZSBTb2xkIikKCiMgQ29sb3JzOiBNYW51YWxseSBtYXRjaGluZyBwbG90IHN0YW5kYXJkIGdyYWRpZW50CnZlcnlfbG93IDwtICIjNDYwZjVjIgpsb3cgPC0gIiMyYzcyOGUiCm1lZCA8LSAiIzI3YWQ4MSIKaGlnaCA8LSAiI2Y0ZTYxZSIKCnBkcF8zZCA8LSBwbG90X2x5KHBkcF9tYXQsIHggPSB+aW5mZWN0aW9uc18zZCwgeSA9IH5hcmVhX2xpdmluZ18zZCwgeiA9IH55aGF0XzNkLAogICAgICAgICAgICAgdHlwZSA9ICdtZXNoM2QnLCBpbnRlbnNpdHkgPSB+eWhhdF8zZCwgCiAgICAgICAgICAgIGNvbG9ycyA9IGNvbG9yUmFtcChjKHZlcnlfbG93LCBtZWQsIGhpZ2gpKSkKcGRwXzNkIDwtIHBkcF8zZCAlPiUgIGxheW91dChzY2VuZSA9IGxpc3QoeGF4aXM9YXh4LHlheGlzPWF4eSx6YXhpcz1heHopKSAjIEF4aXMgbGFicwpwZHBfM2QgPC0gaGlkZV9jb2xvcmJhcihwZHBfM2QpICMgSGlkZSBsZWdlbmQKCiMgUHJpbnQgb3V0IApncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHBfaGVhdF9jb24sIG5yb3cgPSAxKQpwZHBfM2QKCnN0b3BDbHVzdGVyKGNsKQpgYGAKCiMjIyMjIDUuMi40IERheXMgT24gTWFya2V0CiMjIyMjIyA1LjIuNC4xIFN0YW5kYXJkIFBEUApgYGB7cn0KIyBVc2UgcGFyYWxsZWwgY29tcHV0aW5nIHRvIHNwZWVkIHVwIHByb2Nlc3NpbmcgdGltZQpjbCA8LSBtYWtlUFNPQ0tjbHVzdGVyKDUpCnJlZ2lzdGVyRG9QYXJhbGxlbChjbCkKCnBkcCA8LSB4Z2IuZml0LmZpbmFsICU+JQogIHBhcnRpYWwocHJlZC52YXIgPSAiZG9tIiwgbi50cmVlcyA9IDE4MjcsIGdyaWQucmVzb2x1dGlvbiA9IDEwMCwgdHJhaW4gPSBmZWF0dXJlc190cmFpbikgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgdHJhaW4gPSBmZWF0dXJlc190cmFpbikgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmRvbGxhcikgKwogIGdndGl0bGUoIlBEUCIpCgppY2UgPC0geGdiLmZpdC5maW5hbCAlPiUKICBwYXJ0aWFsKHByZWQudmFyID0gImRvbSIsIG4udHJlZXMgPSAxODI3LCBncmlkLnJlc29sdXRpb24gPSAxMDAsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sIGljZSA9IFRSVUUpICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sIGFscGhhID0gLjEsIGNlbnRlciA9IFRSVUUpICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpkb2xsYXIpICsKICBnZ3RpdGxlKCJJQ0UiKQoKZ3JpZEV4dHJhOjpncmlkLmFycmFuZ2UocGRwLCBpY2UsIG5yb3cgPSAyKQoKc3RvcENsdXN0ZXIoY2wpCmBgYAoKCiMjIyMjIyA1LjIuNC4xIE11bHRpX1ZhcmlhYmxlIFBEUApgYGB7cn0KIyBVc2UgcGFyYWxsZWwgY29tcHV0aW5nIHRvIHNwZWVkIHVwIHByb2Nlc3NpbmcgdGltZQpjbCA8LSBtYWtlUFNPQ0tjbHVzdGVyKDUpCnJlZ2lzdGVyRG9QYXJhbGxlbChjbCkKCiMgSGVhdG1hcCB3aXRoIENvbnRvdXIKcGRwX2hlYXRfY29uIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9IGMoImluZmVjdGlvbnNfM21tYSIsICJkb20iKSwgY2h1bGwgPSBUUlVFLCAKICAgICAgICAgIHByb2dyZXNzID0gInRleHQiLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLCBsZXZlbHBsb3QgPSBGQUxTRSkgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgCiAgICAgICAgICAgY29udG91ciA9IFRSVUUsIAogICAgICAgICAgIGNvbnRvdXIuY29sb3IgPSAiI0YwRkZGMCIsIAogICAgICAgICAgIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sCiAgICAgICAgICAgbWFpbiA9ICJJbmZlY3Rpb25zIGFuZCBEYXlzIG9uIE1hcmtldCIsCiAgICAgICAgICAgeGxhYiA9ICJEYWlseSBJbmZlY3Rpb25zICgzbW1hKSIsCiAgICAgICAgICAgeWxhYiA9ICJEYXlzIG9uIE1hcmtldCIsCiAgICAgICAgICAgbGVnZW5kLnRpdGxlID0gIlNvbGQgUHJpY2UiCiAgICAgICAgICAgKQoKIyAzRCBHcmFwaGluZwojIENyZWF0ZSAzRCBkYXRhIG1hdHJpeCAKaW5mZWN0aW9uc18zZCA8LSBwZHBfaGVhdF9jb24kZGF0YSRpbmZlY3Rpb25zXzNtbWEKZG9tXzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJGRvbQp5aGF0XzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJHloYXQKcGRwX21hdCA8LSBkYXRhLmZyYW1lKGluZmVjdGlvbnNfM2QsIGRvbV8zZCwgeWhhdF8zZCkgIyBEYXRhZnJhbSBmb3IgcGxvdGx5IDNEIG1vZGVsCgojIEF4aXMgVGl0bGVzCmF4eCA8LSBsaXN0KHRpdGxlID0gIkluZmVjdGlvbnMgRGFpbHkiKQpheHkgPC0gbGlzdCh0aXRsZSA9ICJEYXlzIG9uIE1hcmtldCIpCmF4eiA8LSBsaXN0KHRpdGxlID0gIlByaWNlIFNvbGQiKQoKIyBDb2xvcnM6IE1hbnVhbGx5IG1hdGNoaW5nIHBsb3Qgc3RhbmRhcmQgZ3JhZGllbnQKdmVyeV9sb3cgPC0gIiM0NjBmNWMiCmxvdyA8LSAiIzJjNzI4ZSIKbWVkIDwtICIjMjdhZDgxIgpoaWdoIDwtICIjZjRlNjFlIgoKcGRwXzNkIDwtIHBsb3RfbHkocGRwX21hdCwgeCA9IH5pbmZlY3Rpb25zXzNkLCB5ID0gfmRvbV8zZCwgeiA9IH55aGF0XzNkLAogICAgICAgICAgICAgdHlwZSA9ICdtZXNoM2QnLCBpbnRlbnNpdHkgPSB+eWhhdF8zZCwgCiAgICAgICAgICAgIGNvbG9ycyA9IGNvbG9yUmFtcChjKHZlcnlfbG93LCBtZWQsIGhpZ2gpKSkKcGRwXzNkIDwtIHBkcF8zZCAlPiUgIGxheW91dChzY2VuZSA9IGxpc3QoeGF4aXM9YXh4LHlheGlzPWF4eSx6YXhpcz1heHopKSAjIEF4aXMgbGFicwpwZHBfM2QgPC0gaGlkZV9jb2xvcmJhcihwZHBfM2QpICMgSGlkZSBsZWdlbmQKCiMgUHJpbnQgb3V0IApncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHBfaGVhdF9jb24sIG5yb3cgPSAxKQpwZHBfM2QKCnN0b3BDbHVzdGVyKGNsKQpgYGAKCiMjIyMjIDUuMi41IEFnZQojIyMjIyMgNS4yLjQuMSBTdGFuZGFyZCBQRFAKYGBge3J9CiMgVXNlIHBhcmFsbGVsIGNvbXB1dGluZyB0byBzcGVlZCB1cCBwcm9jZXNzaW5nIHRpbWUKY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgpwZHAgPC0geGdiLmZpdC5maW5hbCAlPiUKICBwYXJ0aWFsKHByZWQudmFyID0gImFnZSIsIG4udHJlZXMgPSAxODI3LCBncmlkLnJlc29sdXRpb24gPSAxMDAsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4pICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIHRyYWluID0gZmVhdHVyZXNfdHJhaW4pICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpkb2xsYXIpICsKICBnZ3RpdGxlKCJQRFAiKQoKaWNlIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9ICJhZ2UiLCBuLnRyZWVzID0gMTgyNywgZ3JpZC5yZXNvbHV0aW9uID0gMTAwLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLCBpY2UgPSBUUlVFKSAlPiUKICBhdXRvcGxvdChydWcgPSBUUlVFLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLCBhbHBoYSA9IC4xLCBjZW50ZXIgPSBUUlVFKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6ZG9sbGFyKSArCiAgZ2d0aXRsZSgiSUNFIikKCmdyaWRFeHRyYTo6Z3JpZC5hcnJhbmdlKHBkcCwgaWNlLCBucm93ID0gMikKCnN0b3BDbHVzdGVyKGNsKQpgYGAKCiMjIyMjIyA1LjIuNC4xIE11bHRpX1ZhcmlhYmxlIFBEUApgYGB7cn0KIyBVc2UgcGFyYWxsZWwgY29tcHV0aW5nIHRvIHNwZWVkIHVwIHByb2Nlc3NpbmcgdGltZQpjbCA8LSBtYWtlUFNPQ0tjbHVzdGVyKDUpCnJlZ2lzdGVyRG9QYXJhbGxlbChjbCkKCiMgSGVhdG1hcCB3aXRoIENvbnRvdXIKcGRwX2hlYXRfY29uIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9IGMoImluZmVjdGlvbnNfM21tYSIsICJhZ2UiKSwgY2h1bGwgPSBUUlVFLCAKICAgICAgICAgIHByb2dyZXNzID0gInRleHQiLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLCBsZXZlbHBsb3QgPSBGQUxTRSkgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgCiAgICAgICAgICAgY29udG91ciA9IFRSVUUsIAogICAgICAgICAgIGNvbnRvdXIuY29sb3IgPSAiI0YwRkZGMCIsIAogICAgICAgICAgIHRyYWluID0gZmVhdHVyZXNfdHJhaW4sCiAgICAgICAgICAgbWFpbiA9ICJJbmZlY3Rpb25zIGFuZCBBZ2Ugb2YgUHJvcGVydHkiLAogICAgICAgICAgIHhsYWIgPSAiRGFpbHkgSW5mZWN0aW9ucyAoM21tYSkiLAogICAgICAgICAgIHlsYWIgPSAiQWdlIG9mIFByb3BlcnR5IiwKICAgICAgICAgICBsZWdlbmQudGl0bGUgPSAiU29sZCBQcmljZSIKICAgICAgICAgICApCgojIDNEIEdyYXBoaW5nCiMgQ3JlYXRlIDNEIGRhdGEgbWF0cml4IAppbmZlY3Rpb25zXzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJGluZmVjdGlvbnNfM21tYQphZ2VfM2QgPC0gcGRwX2hlYXRfY29uJGRhdGEkYWdlCnloYXRfM2QgPC0gcGRwX2hlYXRfY29uJGRhdGEkeWhhdApwZHBfbWF0IDwtIGRhdGEuZnJhbWUoaW5mZWN0aW9uc18zZCwgYWdlXzNkLCB5aGF0XzNkKSAjIERhdGFmcmFtIGZvciBwbG90bHkgM0QgbW9kZWwKCiMgQXhpcyBUaXRsZXMKYXh4IDwtIGxpc3QodGl0bGUgPSAiSW5mZWN0aW9ucyBEYWlseSIpCmF4eSA8LSBsaXN0KHRpdGxlID0gIkFnZSBvZiBQcm9wZXJ0eSIpCmF4eiA8LSBsaXN0KHRpdGxlID0gIlByaWNlIFNvbGQiKQoKIyBDb2xvcnM6IE1hbnVhbGx5IG1hdGNoaW5nIHBsb3Qgc3RhbmRhcmQgZ3JhZGllbnQKdmVyeV9sb3cgPC0gIiM0NjBmNWMiCmxvdyA8LSAiIzJjNzI4ZSIKbWVkIDwtICIjMjdhZDgxIgpoaWdoIDwtICIjZjRlNjFlIgoKcGRwXzNkIDwtIHBsb3RfbHkocGRwX21hdCwgeCA9IH5pbmZlY3Rpb25zXzNkLCB5ID0gfmFnZV8zZCwgeiA9IH55aGF0XzNkLAogICAgICAgICAgICAgdHlwZSA9ICdtZXNoM2QnLCBpbnRlbnNpdHkgPSB+eWhhdF8zZCwgCiAgICAgICAgICAgIGNvbG9ycyA9IGNvbG9yUmFtcChjKHZlcnlfbG93LCBtZWQsIGhpZ2gpKSkKcGRwXzNkIDwtIHBkcF8zZCAlPiUgIGxheW91dChzY2VuZSA9IGxpc3QoeGF4aXM9YXh4LHlheGlzPWF4eSx6YXhpcz1heHopKSAjIEF4aXMgbGFicwpwZHBfM2QgPC0gaGlkZV9jb2xvcmJhcihwZHBfM2QpICMgSGlkZSBsZWdlbmQKCiMgUHJpbnQgb3V0IApncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHBfaGVhdF9jb24sIG5yb3cgPSAxKQpwZHBfM2QKCnN0b3BDbHVzdGVyKGNsKQpgYGAKCgojIyMjIyA1LjIuNiBTb2xkIERhdGUKIyMjIyMjIDUuMi40LjEgU3RhbmRhcmQgUERQCmBgYHtyfQojIFVzZSBwYXJhbGxlbCBjb21wdXRpbmcgdG8gc3BlZWQgdXAgcHJvY2Vzc2luZyB0aW1lCmNsIDwtIG1ha2VQU09DS2NsdXN0ZXIoNSkKcmVnaXN0ZXJEb1BhcmFsbGVsKGNsKQoKcGRwIDwtIHhnYi5maXQuZmluYWwgJT4lCiAgcGFydGlhbChwcmVkLnZhciA9ICJzb2xkX2RhdGUiLCBuLnRyZWVzID0gMTgyNywgZ3JpZC5yZXNvbHV0aW9uID0gMTAwLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluKSAlPiUKICBhdXRvcGxvdChydWcgPSBUUlVFLCB0cmFpbiA9IGZlYXR1cmVzX3RyYWluKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6ZG9sbGFyKSArCiAgZ2d0aXRsZSgiUERQIikKCmljZSA8LSB4Z2IuZml0LmZpbmFsICU+JQogIHBhcnRpYWwocHJlZC52YXIgPSAic29sZF9kYXRlIiwgbi50cmVlcyA9IDE4MjcsIGdyaWQucmVzb2x1dGlvbiA9IDEwMCwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgaWNlID0gVFJVRSkgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgYWxwaGEgPSAuMSwgY2VudGVyID0gVFJVRSkgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmRvbGxhcikgKwogIGdndGl0bGUoIklDRSIpCgpncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHAsIGljZSwgbnJvdyA9IDIpCgpzdG9wQ2x1c3RlcihjbCkKYGBgCgojIyMjIyMgNS4yLjQuMSBNdWx0aV9WYXJpYWJsZSBQRFAKYGBge3J9CiMgVXNlIHBhcmFsbGVsIGNvbXB1dGluZyB0byBzcGVlZCB1cCBwcm9jZXNzaW5nIHRpbWUKY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgojIEhlYXRtYXAgd2l0aCBDb250b3VyCnBkcF9oZWF0X2NvbiA8LSB4Z2IuZml0LmZpbmFsICU+JQogIHBhcnRpYWwocHJlZC52YXIgPSBjKCJpbmZlY3Rpb25zXzNtbWEiLCAic29sZF9kYXRlIiksIGNodWxsID0gVFJVRSwgCiAgICAgICAgICBwcm9ncmVzcyA9ICJ0ZXh0IiwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgbGV2ZWxwbG90ID0gRkFMU0UpICU+JQogIGF1dG9wbG90KHJ1ZyA9IFRSVUUsIAogICAgICAgICAgIGNvbnRvdXIgPSBUUlVFLCAKICAgICAgICAgICBjb250b3VyLmNvbG9yID0gIiNGMEZGRjAiLCAKICAgICAgICAgICB0cmFpbiA9IGZlYXR1cmVzX3RyYWluLAogICAgICAgICAgIG1haW4gPSAiSW5mZWN0aW9ucyBhbmQgRGF0ZSBTb2xkIiwKICAgICAgICAgICB4bGFiID0gIkRhaWx5IEluZmVjdGlvbnMgKDNtbWEpIiwKICAgICAgICAgICB5bGFiID0gIkRhdGUgU29sZCIsCiAgICAgICAgICAgbGVnZW5kLnRpdGxlID0gIlNvbGQgUHJpY2UiCiAgICAgICAgICAgKQoKIyAzRCBHcmFwaGluZwojIENyZWF0ZSAzRCBkYXRhIG1hdHJpeCAKaW5mZWN0aW9uc18zZCA8LSBwZHBfaGVhdF9jb24kZGF0YSRpbmZlY3Rpb25zXzNtbWEKc29sZF9kYXRlXzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJHNvbGRfZGF0ZQp5aGF0XzNkIDwtIHBkcF9oZWF0X2NvbiRkYXRhJHloYXQKcGRwX21hdCA8LSBkYXRhLmZyYW1lKGluZmVjdGlvbnNfM2QsIHNvbGRfZGF0ZV8zZCwgeWhhdF8zZCkgIyBEYXRhZnJhbSBmb3IgcGxvdGx5IDNEIG1vZGVsCgojIEF4aXMgVGl0bGVzCmF4eCA8LSBsaXN0KHRpdGxlID0gIkluZmVjdGlvbnMgRGFpbHkiKQpheHkgPC0gbGlzdCh0aXRsZSA9ICJEYXRlIFNvbGQiKQpheHogPC0gbGlzdCh0aXRsZSA9ICJQcmljZSBTb2xkIikKCiMgQ29sb3JzOiBNYW51YWxseSBtYXRjaGluZyBwbG90IHN0YW5kYXJkIGdyYWRpZW50CnZlcnlfbG93IDwtICIjNDYwZjVjIgpsb3cgPC0gIiMyYzcyOGUiCm1lZCA8LSAiIzI3YWQ4MSIKaGlnaCA8LSAiI2Y0ZTYxZSIKCnBkcF8zZCA8LSBwbG90X2x5KHBkcF9tYXQsIHggPSB+aW5mZWN0aW9uc18zZCwgeSA9IH5zb2xkX2RhdGVfM2QsIHogPSB+eWhhdF8zZCwKICAgICAgICAgICAgIHR5cGUgPSAnbWVzaDNkJywgaW50ZW5zaXR5ID0gfnloYXRfM2QsIAogICAgICAgICAgICBjb2xvcnMgPSBjb2xvclJhbXAoYyh2ZXJ5X2xvdywgbWVkLCBoaWdoKSkpCnBkcF8zZCA8LSBwZHBfM2QgJT4lICBsYXlvdXQoc2NlbmUgPSBsaXN0KHhheGlzPWF4eCx5YXhpcz1heHksemF4aXM9YXh6KSkgIyBBeGlzIGxhYnMKcGRwXzNkIDwtIGhpZGVfY29sb3JiYXIocGRwXzNkKSAjIEhpZGUgbGVnZW5kCgojIFByaW50IG91dCAKZ3JpZEV4dHJhOjpncmlkLmFycmFuZ2UocGRwX2hlYXRfY29uLCBucm93ID0gMSkKcGRwXzNkCgpzdG9wQ2x1c3RlcihjbCkKYGBgCgojIyMjIyA1LjIuNyBQaG90byBDb3VudAojIyMjIyMgNS4yLjQuMSBTdGFuZGFyZCBQRFAKYGBge3J9CiMgVXNlIHBhcmFsbGVsIGNvbXB1dGluZyB0byBzcGVlZCB1cCBwcm9jZXNzaW5nIHRpbWUKY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgpwZHAgPC0geGdiLmZpdC5maW5hbCAlPiUKICBwYXJ0aWFsKHByZWQudmFyID0gInBob3RvX2NvdW50Iiwgbi50cmVlcyA9IDE4MjcsIGdyaWQucmVzb2x1dGlvbiA9IDEwMCwgdHJhaW4gPSBmZWF0dXJlc190cmFpbikgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgdHJhaW4gPSBmZWF0dXJlc190cmFpbikgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmRvbGxhcikgKwogIGdndGl0bGUoIlBEUCIpCgppY2UgPC0geGdiLmZpdC5maW5hbCAlPiUKICBwYXJ0aWFsKHByZWQudmFyID0gInBob3RvX2NvdW50Iiwgbi50cmVlcyA9IDE4MjcsIGdyaWQucmVzb2x1dGlvbiA9IDEwMCwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgaWNlID0gVFJVRSkgJT4lCiAgYXV0b3Bsb3QocnVnID0gVFJVRSwgdHJhaW4gPSBmZWF0dXJlc190cmFpbiwgYWxwaGEgPSAuMSwgY2VudGVyID0gVFJVRSkgKwogIHNjYWxlX3lfY29udGludW91cyhsYWJlbHMgPSBzY2FsZXM6OmRvbGxhcikgKwogIGdndGl0bGUoIklDRSIpCgpncmlkRXh0cmE6OmdyaWQuYXJyYW5nZShwZHAsIGljZSwgbnJvdyA9IDIpCgpzdG9wQ2x1c3RlcihjbCkKYGBgCgoKIyMjIyMgNS4yLjIgUHJpY2UKIyMjIyMgNS4yLjIgTnVtYmVyIG9mIEJlZHJvb21zCgoKIyMjIFBsYXlncm91bmQKTElNRQpgYGB7cn0KIyBvbmUtaG90IGVuY29kZSB0aGUgbG9jYWwgb2JzZXJ2YXRpb25zIHRvIGJlIGFzc2Vzc2VkLgpsb2NhbF9vYnNfb25laG90IDwtIHZ0cmVhdDo6cHJlcGFyZSh0cmVhdHBsYW4sIGxvY2FsX29icywgdmFyUmVzdHJpY3Rpb24gPSBuZXdfdmFycykKCiMgYXBwbHkgTElNRQpleHBsYWluZXIgPC0gbGltZShkYXRhLmZyYW1lKGZlYXR1cmVzX3RyYWluKSwgeGdiLmZpdC5maW5hbCkKZXhwbGFuYXRpb24gPC0gZXhwbGFpbihsb2NhbF9vYnNfb25laG90LCBleHBsYWluZXIsIG5fZmVhdHVyZXMgPSA1KQpwbG90X2ZlYXR1cmVzKGV4cGxhbmF0aW9uKQpgYGAKCgpQcmVkaWN0aW5nIGltcGFjdCBvZiBoaWdoZXIgY2FzZXMgb2YgY29yb25hCmBgYHtyfQojIHByZWRpY3QgdmFsdWVzIGZvciB0ZXN0IGRhdGEKCmZlYXR1cmVfdGVzdCA8LSBmZWF0dXJlc190ZXN0WzE6MixdCgpwcmVkIDwtIHByZWRpY3QoeGdiLmZpdC5maW5hbCwgZmVhdHVyZV90ZXN0KQpwcmVkCgo/cHJlZGljdAoKIyByZXN1bHRzCmNhcmV0OjpSTVNFKHByZWQsIHJlc3BvbnNlX3Rlc3QpCiMjIFsxXSAyMTMxOS4zCmBgYAoKCgpgYGB7cn0KY2wgPC0gbWFrZVBTT0NLY2x1c3Rlcig1KQpyZWdpc3RlckRvUGFyYWxsZWwoY2wpCgojIHJlcHJvZHVjaWJpbGl0eQpzZXQuc2VlZCgyKQoKc3RhcnRfdGltZSA8LSBwcm9jLnRpbWUoKQp4Z2IuZml0LmZpbmFsIDwtIHhnYm9vc3QoCiAgZGF0YSA9IGZlYXR1cmVzX3RyYWluLAogIGxhYmVsID0gcmVzcG9uc2VfdHJhaW4sCiAgbnJvdW5kcyA9IDk1MCwKICBuZm9sZCA9IDUsCiAgb2JqZWN0aXZlID0gInJlZzpsaW5lYXIiLCAgICMgZm9yIHJlZ3Jlc3Npb24gbW9kZWxzCiAgdmVyYm9zZSA9IDAsICAgICAgICAgICAgICAgICMgc2lsZW50LAogIGVhcmx5X3N0b3BwaW5nX3JvdW5kcyA9IDEwICAjIHN0b3AgaWYgbm8gaW1wcm92ZW1lbnQgZm9yIDEwIGNvbnNlY3V0aXZlIHRyZWVzCikKZW5kX3RpbWUgPC0gcHJvYy50aW1lKCkKdGltZV90YWtlbiA8LSBlbmRfdGltZSAtIHN0YXJ0X3RpbWUKdGltZV90YWtlbgoKc3RvcENsdXN0ZXIoY2wpCmBgYAoKCgpFbmQgb2YgQ29kZQoKCgoKCg==