Deep Learning with h20
Nana Boateng
October 08, 2017
The data consist if surveys administered to people who live near a community health facilty. The goal here is to predict the rate of hypertension among the residents given predictors such as age,race,gender,education and employment status.
rm(list =c())
library(h2o)
library(h2oEnsemble)
library(tidyverse)
h2o.init(nthreads = -1, #Number of threads -1 means use all cores on your machine
max_mem_size = "8G") #max mem size is the maximum memory to allocate to H2O Connection successful!
R is connected to the H2O cluster:
H2O cluster uptime: 15 minutes 23 seconds
H2O cluster version: 3.14.0.3
H2O cluster version age: 15 days
H2O cluster name: H2O_started_from_R_nanaakwasiabayieboateng_qlb223
H2O cluster total nodes: 1
H2O cluster total memory: 6.70 GB
H2O cluster total cores: 8
H2O cluster allowed cores: 8
H2O cluster healthy: TRUE
H2O Connection ip: localhost
H2O Connection port: 54321
H2O Connection proxy: NA
H2O Internal Security: FALSE
H2O API Extensions: XGBoost, Algos, AutoML, Core V3, Core V4
R Version: R version 3.4.1 (2017-06-30) localH2O = h2o.init(ip = 'localhost', port = 54321, nthreads = -1,max_mem_size = "8G") Connection successful!
R is connected to the H2O cluster:
H2O cluster uptime: 15 minutes 23 seconds
H2O cluster version: 3.14.0.3
H2O cluster version age: 15 days
H2O cluster name: H2O_started_from_R_nanaakwasiabayieboateng_qlb223
H2O cluster total nodes: 1
H2O cluster total memory: 6.70 GB
H2O cluster total cores: 8
H2O cluster allowed cores: 8
H2O cluster healthy: TRUE
H2O Connection ip: localhost
H2O Connection port: 54321
H2O Connection proxy: NA
H2O Internal Security: FALSE
H2O API Extensions: XGBoost, Algos, AutoML, Core V3, Core V4
R Version: R version 3.4.1 (2017-06-30) loan_csv <- "/Users/nanaakwasiabayieboateng/Documents/memphisclassesbooks/DataMiningscience/UICProject/Workbook3.csv"
datachicago <- h2o.importFile(loan_csv)
|
| | 0%
|
|=========================================================================================| 100%dim(datachicago)[1] 454 17head(datachicago)h2o.names((datachicago)) [1] "ID" "Gender" "Age" "Hypertension"
[5] "Employment" "Hispanic" "Education" "Hispanic_ancestry"
[9] "race_white" "race_black" "race_Asian" "race_pacific"
[13] "race_Amer_Indian" "race_other" "race_dontknow" "race_refused"
[17] "Asian_ancestry" str(datachicago)Class 'H2OFrame' <environment: 0x11413cdb8>
- attr(*, "op")= chr "Parse"
- attr(*, "id")= chr "Workbook3.hex_sid_9881_8"
- attr(*, "eval")= logi FALSE
- attr(*, "nrow")= int 454
- attr(*, "ncol")= int 17
- attr(*, "types")=List of 17
..$ : chr "int"
..$ : chr "enum"
..$ : chr "int"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
- attr(*, "data")='data.frame': 10 obs. of 17 variables:
..$ ID : num 1 2 3 4 5 6 7 8 9 10
..$ Gender : Factor w/ 2 levels "Female","Male": 2 1 2 1 1 1 2 1 1 2
..$ Age : num 24 76 47 42 24 39 31 25 33 32
..$ Hypertension : Factor w/ 4 levels "NA","NO CODED RESPONSE APPLICABLE (SPECIFY)",..: 4 4 3 3 3 3 4 3 3 3
..$ Employment : Factor w/ 9 levels "Employed for wages,",..: 8 7 1 1 1 1 1 6 1 6
..$ Hispanic : Factor w/ 3 levels "NO CODED RESPONSE APPLICABLE (SPECIFY)",..: 2 2 2 2 2 2 2 2 2 2
..$ Education : Factor w/ 24 levels "10th Grade","11th Grade",..: 15 19 15 15 15 20 15 15 20 20
..$ Hispanic_ancestry: Factor w/ 7 levels ".","Ecuadorian, or",..: 1 1 1 1 1 1 1 1 1 1
..$ race_white : Factor w/ 2 levels "No","Yes": 2 2 2 2 2 2 2 2 2 2
..$ race_black : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_Asian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_pacific : Factor w/ 1 level "No": 1 1 1 1 1 1 1 1 1 1
..$ race_Amer_Indian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_other : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_dontknow : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_refused : Factor w/ 4 levels ".","Chinese,",..: 3 3 3 3 3 3 3 3 3 3
..$ Asian_ancestry : Factor w/ 5 levels ".","Asian Indian,",..: 1 1 1 1 1 1 1 1 1 1#==========================================================================================================
# Look at the structure of the data with the glimpse function in
# dplyr package
#==========================================================================================================
str(data)Classes ‘data.table’ and 'data.frame': 434 obs. of 17 variables:
$ ID : int 1 2 3 4 5 6 7 8 9 10 ...
$ Gender : chr "Male" "Female" "Male" "Female" ...
$ Age : num 24 76 47 42 24 39 31 25 33 32 ...
$ Hypertension : chr "Yes" "Yes" "No" "No" ...
$ Employment : chr "Self-employed," "Primarily retired, or" "Employed for wages," "Employed for wages," ...
$ Hispanic : chr "No" "No" "No" "No" ...
$ Education : chr "Bachelor's Degree (Example: BA, AB, BS, BBA)" "High School Graduate" "Bachelor's Degree (Example: BA, AB, BS, BBA)" "Bachelor's Degree (Example: BA, AB, BS, BBA)" ...
$ Hispanic_ancestry: chr "." "." "." "." ...
$ race_white : chr "Yes" "Yes" "Yes" "Yes" ...
$ race_black : chr "No" "No" "No" "No" ...
$ race_Asian : chr "No" "No" "No" "No" ...
$ race_pacific : chr "No" "No" "No" "No" ...
$ race_Amer_Indian : chr "No" "No" "No" "No" ...
$ race_other : chr "No" "No" "No" "No" ...
$ race_dontknow : chr "No" "No" "No" "No" ...
$ race_refused : chr "No" "No" "No" "No" ...
$ Asian_ancestry : chr "<NA>" "<NA>" "<NA>" "<NA>" ...
- attr(*, ".internal.selfref")=<externalptr> dplyr::glimpse(data)Observations: 434
Variables: 17
$ ID <int> 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 2...
$ Gender <chr> "Male", "Female", "Male", "Female", "Female", "Female", "Male", "Fem...
$ Age <dbl> 24, 76, 47, 42, 24, 39, 31, 25, 33, 32, 24, 34, 26, 31, 24, 26, 34, ...
$ Hypertension <chr> "Yes", "Yes", "No", "No", "No", "No", "Yes", "No", "No", "No", "No",...
$ Employment <chr> "Self-employed,", "Primarily retired, or", "Employed for wages,", "E...
$ Hispanic <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ Education <chr> "Bachelor's Degree (Example: BA, AB, BS, BBA)", "High School Graduat...
$ Hispanic_ancestry <chr> ".", ".", ".", ".", ".", ".", ".", ".", ".", ".", ".", ".", ".", "."...
$ race_white <chr> "Yes", "Yes", "Yes", "Yes", "Yes", "Yes", "Yes", "Yes", "Yes", "Yes"...
$ race_black <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_Asian <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_pacific <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_Amer_Indian <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_other <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_dontknow <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ race_refused <chr> "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "No", "N...
$ Asian_ancestry <chr> "<NA>", "<NA>", "<NA>", "<NA>", "<NA>", "<NA>", "<NA>", "<NA>", "<NA...summary(data) ID Gender Age Hypertension Employment
Min. : 1.0 Length:434 Min. :19.00 Length:434 Length:434
1st Qu.:110.2 Class :character 1st Qu.:29.00 Class :character Class :character
Median :221.5 Mode :character Median :39.00 Mode :character Mode :character
Mean :221.6 Mean :42.49
3rd Qu.:331.8 3rd Qu.:54.75
Max. :444.0 Max. :91.00
Hispanic Education Hispanic_ancestry race_white race_black
Length:434 Length:434 Length:434 Length:434 Length:434
Class :character Class :character Class :character Class :character Class :character
Mode :character Mode :character Mode :character Mode :character Mode :character
race_Asian race_pacific race_Amer_Indian race_other race_dontknow
Length:434 Length:434 Length:434 Length:434 Length:434
Class :character Class :character Class :character Class :character Class :character
Mode :character Mode :character Mode :character Mode :character Mode :character
race_refused Asian_ancestry
Length:434 Length:434
Class :character Class :character
Mode :character Mode :character
#==========================================================================================================
#check the number of missing rows
#==========================================================================================================
colSums(is.na.data.frame(data)) ID Gender Age Hypertension Employment
0 0 0 0 0
Hispanic Education Hispanic_ancestry race_white race_black
0 0 0 0 0
race_Asian race_pacific race_Amer_Indian race_other race_dontknow
0 0 0 0 0
race_refused Asian_ancestry
0 0 data[!complete.cases(data),]%>%head()data[which(data$Asian_ancestry!="."),]data$Asian_ancestry=ifelse(data$Asian_ancestry==".","<NA>",data$Asian_ancestry)
data=data[complete.cases(data),]
#==========================================================================================================
#NO CODED RESPONSE APPLICABLE (SPECIFY)
#==========================================================================================================
#data%>%dplyr::filter(str_detect(Hypertension, "NO CODED RESPONSE APPLICABLE (SPECIFY)"))
ndata=data%>%dplyr::select(-ID)%>%dplyr::filter(Hypertension!="NO CODED RESPONSE APPLICABLE (SPECIFY)",Employment
!="NO CODED RESPONSE APPLICABLE (LEAVE NOTE FIRST)"
,Hispanic!="NO CODED RESPONSE APPLICABLE (SPECIFY)",
Education!="NO CODED RESPONSE APPLICABLE (SPECIFY)")
ndata=mutate_if(ndata,is.character,as.factor)
str(ndata)'data.frame': 424 obs. of 16 variables:
$ Gender : Factor w/ 2 levels "Female","Male": 2 1 2 1 1 1 2 1 1 2 ...
$ Age : num 24 76 47 42 24 39 31 25 33 32 ...
$ Hypertension : Factor w/ 2 levels "No","Yes": 2 2 1 1 1 1 2 1 1 1 ...
$ Employment : Factor w/ 8 levels "Employed for wages,",..: 7 6 1 1 1 1 1 5 1 5 ...
$ Hispanic : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ Education : Factor w/ 23 levels "10th Grade","11th Grade",..: 15 19 15 15 15 20 15 15 20 20 ...
$ Hispanic_ancestry: Factor w/ 5 levels ".","Ecuadorian, or",..: 1 1 1 1 1 1 1 1 1 1 ...
$ race_white : Factor w/ 2 levels "No","Yes": 2 2 2 2 2 2 2 2 2 2 ...
$ race_black : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ race_Asian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ race_pacific : Factor w/ 1 level "No": 1 1 1 1 1 1 1 1 1 1 ...
$ race_Amer_Indian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ race_other : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ race_dontknow : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ race_refused : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1 ...
$ Asian_ancestry : Factor w/ 4 levels "<NA>","Asian Indian,",..: 1 1 1 1 1 1 1 1 1 1 ...#==========================================================================================================
# import R object to the H2O cloud.
#convert r data to h2o object
#==========================================================================================================
datah20=as.h2o(ndata)
|
| | 0%
|
|=========================================================================================| 100%str(datah20)Class 'H2OFrame' <environment: 0x10ce545c8>
- attr(*, "op")= chr "Parse"
- attr(*, "id")= chr "ndata"
- attr(*, "eval")= logi FALSE
- attr(*, "nrow")= int 424
- attr(*, "ncol")= int 16
- attr(*, "types")=List of 16
..$ : chr "enum"
..$ : chr "int"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
..$ : chr "enum"
- attr(*, "data")='data.frame': 10 obs. of 16 variables:
..$ Gender : Factor w/ 2 levels "Female","Male": 2 1 2 1 1 1 2 1 1 2
..$ Age : num 24 76 47 42 24 39 31 25 33 32
..$ Hypertension : Factor w/ 2 levels "No","Yes": 2 2 1 1 1 1 2 1 1 1
..$ Employment : Factor w/ 8 levels "Employed for wages,",..: 7 6 1 1 1 1 1 5 1 5
..$ Hispanic : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ Education : Factor w/ 23 levels "10th Grade","11th Grade",..: 15 19 15 15 15 20 15 15 20 20
..$ Hispanic_ancestry: Factor w/ 5 levels ".","Ecuadorian, or",..: 1 1 1 1 1 1 1 1 1 1
..$ race_white : Factor w/ 2 levels "No","Yes": 2 2 2 2 2 2 2 2 2 2
..$ race_black : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_Asian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_pacific : Factor w/ 1 level "No": 1 1 1 1 1 1 1 1 1 1
..$ race_Amer_Indian : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_other : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_dontknow : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ race_refused : Factor w/ 2 levels "No","Yes": 1 1 1 1 1 1 1 1 1 1
..$ Asian_ancestry : Factor w/ 4 levels "<NA>","Asian Indian,",..: 1 1 1 1 1 1 1 1 1 1#==========================================================================================================
# Partition the data into training, validation and test sets
#==========================================================================================================
splits <- h2o.splitFrame(data = datah20,
ratios = c(0.7, 0.15), #partition data into 70%, 15%, 15% chunks
seed = 1) #setting a seed will guarantee reproducibility
train <- splits[[1]]
valid <- splits[[2]]
test <- splits[[3]]
# Identify response and predictor variables
y <- "Hypertension"
x <- setdiff(names(datah20), y) #==========================================================================================================
#glm/logistic
#similar to R's glm, h2o.glm has the family argument
# 1. Let's start with a basic binomial Generalized Linear Model
# By default, h2o.glm uses a regularized, elastic net model
#==========================================================================================================
glm_fit1 <- h2o.glm(x = x,
y = y,
training_frame = train,
model_id = "glm_fit1",
family = "binomial") Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|=========================================================================================| 100%#====================================================================================================================================
# Next we will do some automatic tuning by passing in a validation frame and setting
# `lambda_search = True`. Since we are training a GLM with regularization, we should
# try to find the right amount of regularization (to avoid overfitting). The model
# parameter, `lambda`, controls the amount of regularization in a GLM model and we can
# find the optimal value for `lambda` automatically by setting `lambda_search = TRUE`
# and passing in a validation frame (which is used to evaluate model performance using a
# particular value of lambda).
#=====================================================================================================================================
glm_fit2 <- h2o.glm(x = x,
y = y,
training_frame = train,
model_id = "glm_fit2",
validation_frame = valid,
family = "binomial",
lambda_search = TRUE)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|================= | 19%
|
|=========================================================================================| 100%#==========================================================================================================
# Let's compare the performance of the two GLMs
#==========================================================================================================
glm_perf1 <- h2o.performance(model = glm_fit1,
newdata = test)
glm_perf2 <- h2o.performance(model = glm_fit2,
newdata = test)
(glm_perf1) H2OBinomialMetrics: glm
MSE: 0.2021788
RMSE: 0.449643
LogLoss: 0.5969429
Mean Per-Class Error: 0.3113984
AUC: 0.7079906
Gini: 0.4159812
R^2: 0.1447194
Residual Deviance: 71.63315
AIC: 79.63315
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 22 15 0.405405 =15/37
Yes 5 18 0.217391 =5/23
Totals 27 33 0.333333 =20/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.239391 0.642857 27
2 max f2 0.131403 0.782313 40
3 max f0point5 0.550961 0.714286 9
4 max accuracy 0.550961 0.750000 9
5 max precision 0.728177 1.000000 0
6 max recall 0.131403 1.000000 40
7 max specificity 0.728177 1.000000 0
8 max absolute_mcc 0.550961 0.475239 9
9 max min_per_class_accuracy 0.320071 0.648649 22
10 max mean_per_class_accuracy 0.273119 0.693890 24
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`(glm_perf2) H2OBinomialMetrics: glm
MSE: 0.1965816
RMSE: 0.4433753
LogLoss: 0.5866738
Mean Per-Class Error: 0.3143361
AUC: 0.7285546
Gini: 0.4571093
R^2: 0.1683973
Residual Deviance: 70.40085
AIC: 98.40085
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 25 12 0.324324 =12/37
Yes 7 16 0.304348 =7/23
Totals 32 28 0.316667 =19/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.282829 0.627451 27
2 max f2 0.088373 0.782313 53
3 max f0point5 0.397360 0.689655 15
4 max accuracy 0.397360 0.750000 15
5 max precision 0.814130 1.000000 0
6 max recall 0.088373 1.000000 53
7 max specificity 0.814130 1.000000 0
8 max absolute_mcc 0.397360 0.454770 15
9 max min_per_class_accuracy 0.282829 0.675676 27
10 max mean_per_class_accuracy 0.397360 0.706816 15
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`#==========================================================================================================
# Instead of printing the entire model performance metrics object,
# it is probably easier to print just the metric that you are interested in comparing.
# Retreive test set AUC
#==========================================================================================================
h2o.auc(glm_perf1) [1] 0.7079906h2o.auc(glm_perf2) [1] 0.7285546#==========================================================================================================
# Compare test AUC to the training AUC and validation AUC
#==========================================================================================================
h2o.auc(glm_fit2, train = TRUE) [1] 0.8268307h2o.auc(glm_fit2, valid = TRUE) [1] 0.8009756glm_fit2@model$validation_metrics H2OBinomialMetrics: glm
** Reported on validation data. **
MSE: 0.175042
RMSE: 0.4183802
LogLoss: 0.522122
Mean Per-Class Error: 0.2443902
AUC: 0.8009756
Gini: 0.6019512
R^2: 0.2561143
Residual Deviance: 68.9201
AIC: 96.9201
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 39 2 0.048780 =2/41
Yes 11 14 0.440000 =11/25
Totals 50 16 0.196970 =13/66
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.656748 0.682927 15
2 max f2 0.176934 0.789474 51
3 max f0point5 0.706704 0.821918 11
4 max accuracy 0.706704 0.803030 11
5 max precision 0.917858 1.000000 0
6 max recall 0.124401 1.000000 58
7 max specificity 0.917858 1.000000 0
8 max absolute_mcc 0.706704 0.603692 11
9 max min_per_class_accuracy 0.467950 0.720000 28
10 max mean_per_class_accuracy 0.656748 0.755610 15
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`#==========================================================================================================
# 2. Random Forest
# H2O's Random Forest (RF) implements a distributed version of the standard
# Random Forest algorithm and variable importance measures.
# First we will train a basic Random Forest model with default parameters.
# The Random Forest model will infer the response distribution from the response encoding.
# A seed is required for reproducibility.
#==========================================================================================================
rf_fit1 <- h2o.randomForest(x = x,
y = y,
training_frame = train,
model_id = "rf_fit1",
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|================== | 20%
|
|=========================================================================================| 100%#==========================================================================================================
# Next we will increase the number of trees used in the forest by setting `ntrees = 100`.
# The default number of trees in an H2O Random Forest is 50, so this RF will be twice as
# big as the default. Usually increasing the number of trees in a RF will increase
# performance as well. Unlike Gradient Boosting Machines (GBMs), Random Forests are fairly
# resistant (although not free from) overfitting.
# See the GBM example below for additional guidance on preventing overfitting using H2O's
# early stopping functionality.
#==========================================================================================================
rf_fit2 <- h2o.randomForest(x = x,
y = y,
training_frame = train,
model_id = "rf_fit2",
#validation_frame = valid, #only used if stopping_rounds > 0
ntrees = 100,
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|===== | 6%
|
|================================================================ | 72%
|
|=========================================================================================| 100%#==========================================================================================================
# Let's compare the performance of the two RFs
#==========================================================================================================
rf_perf1 <- h2o.performance(model = rf_fit1,
newdata = test)
rf_perf2 <- h2o.performance(model = rf_fit2,
newdata = test)#==========================================================================================================
# Print model performance
#==========================================================================================================
rf_perf1H2OBinomialMetrics: drf
MSE: 0.2063654
RMSE: 0.4542746
LogLoss: 0.6252983
Mean Per-Class Error: 0.2984724
AUC: 0.6921269
Gini: 0.3842538
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 31 6 0.162162 =6/37
Yes 10 13 0.434783 =10/23
Totals 41 19 0.266667 =16/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.365918 0.619048 18
2 max f2 0.049115 0.761589 58
3 max f0point5 0.365918 0.656566 18
4 max accuracy 0.365918 0.733333 18
5 max precision 0.852486 1.000000 0
6 max recall 0.049115 1.000000 58
7 max specificity 0.852486 1.000000 0
8 max absolute_mcc 0.615149 0.422781 5
9 max min_per_class_accuracy 0.300039 0.652174 25
10 max mean_per_class_accuracy 0.365918 0.701528 18
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`rf_perf2H2OBinomialMetrics: drf
MSE: 0.2098127
RMSE: 0.4580531
LogLoss: 0.6416177
Mean Per-Class Error: 0.3225617
AUC: 0.6780259
Gini: 0.3560517
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 26 11 0.297297 =11/37
Yes 8 15 0.347826 =8/23
Totals 34 26 0.316667 =19/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.294974 0.612245 25
2 max f2 0.033693 0.756579 59
3 max f0point5 0.626739 0.638298 5
4 max accuracy 0.626739 0.716667 5
5 max precision 0.810297 1.000000 0
6 max recall 0.033693 1.000000 59
7 max specificity 0.810297 1.000000 0
8 max absolute_mcc 0.626739 0.422781 5
9 max min_per_class_accuracy 0.294974 0.652174 25
10 max mean_per_class_accuracy 0.420257 0.679788 17
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`#==========================================================================================================
# Retreive test set AUC
#==========================================================================================================
h2o.auc(rf_perf1) [1] 0.6921269h2o.auc(rf_perf2) [1] 0.6780259#==========================================================================================================
# Cross-validate performance
# Rather than using held-out test set to evaluate model performance, a user may wish
# to estimate model performance using cross-validation. Using the RF algorithm
# (with default model parameters) as an example, we demonstrate how to perform k-fold
# cross-validation using H2O. No custom code or loops are required, you simply specify
# the number of desired folds in the nfolds argument.
# Since we are not going to use a test set here, we can use the original (full) dataset,
# which we called data rather than the subsampled `train` dataset. Note that this will
# take approximately k (nfolds) times longer than training a single RF model, since it
# will train k models in the cross-validation process (trained on n(k-1)/k rows), in
# addition to the final model trained on the full training_frame dataset with n rows.
#==========================================================================================================
rf_fit3 <- h2o.randomForest(x = x,
y = y,
training_frame = train,
model_id = "rf_fit3",
seed = 1,
nfolds = 5)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|====== | 7%
|
|========================================== | 47%
|
|===================================================================== | 78%
|
|=========================================================================================| 100%#==========================================================================================================
# To evaluate the cross-validated AUC, do the following:
#==========================================================================================================
h2o.auc(rf_fit3, xval = TRUE) [1] 0.789991#==========================================================================================================
# 3. Gradient Boosting Machine
# H2O's Gradient Boosting Machine (GBM) offers a Stochastic GBM, which can
# increase performance quite a bit compared to the original GBM implementation.
# Now we will train a basic GBM model
# The GBM model will infer the response distribution from the response encoding if not specified
# explicitly through the `distribution` argument. A seed is required for reproducibility.
#==========================================================================================================
gbm_fit1 <- h2o.gbm(x = x,
y = y,
training_frame = train,
model_id = "gbm_fit1",
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|============ | 14%
|
|=========================================================================================| 100%#==========================================================================================================
# Next we will increase the number of trees used in the GBM by setting `ntrees=500`.
# The default number of trees in an H2O GBM is 50, so this GBM will trained using ten times
# the default. Increasing the number of trees in a GBM is one way to increase performance
# of the model, however, you have to be careful not to overfit your model to the training data
# by using too many trees. To automatically find the optimal number of trees, you must use
# H2O's early stopping functionality. This example will not do that, however, the following
# example will.
#==========================================================================================================
gbm_fit2 <- h2o.gbm(x = x,
y = y,
training_frame = train,
model_id = "gbm_fit2",
#validation_frame = valid, #only used if stopping_rounds > 0
ntrees = 500,
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|= | 1%
|
|============= | 15%
|
|==================== | 23%
|
|========================== | 29%
|
|===================================== | 41%
|
|=========================================================================================| 100%#============================================================================================
# We will again set `ntrees = 500`, however, this time we will use early stopping in order to
# prevent overfitting (from too many trees). All of H2O's algorithms have early stopping available,
# however early stopping is not enabled by default (with the exception of Deep Learning).
# There are several parameters that should be used to control early stopping. The three that are
# common to all the algorithms are: `stopping_rounds`, `stopping_metric` and `stopping_tolerance`.
# The stopping metric is the metric by which you'd like to measure performance, and so we will choose
# AUC here. The `score_tree_interval` is a parameter specific to the Random Forest model and the GBM.
# Setting `score_tree_interval = 5` will score the model after every five trees. The parameters we
# have set below specify that the model will stop training after there have been three scoring intervals
# where the AUC has not increased more than 0.0005. Since we have specified a validation frame,
# the stopping tolerance will be computed on validation AUC rather than training AUC.
#===============================================================================================
gbm_fit3 <- h2o.gbm(x = x,
y = y,
training_frame = train,
model_id = "gbm_fit3",
validation_frame = valid, #only used if stopping_rounds > 0
ntrees = 500,
score_tree_interval = 5, #used for early stopping
stopping_rounds = 3, #used for early stopping
stopping_metric = "AUC", #used for early stopping
stopping_tolerance = 0.0005, #used for early stopping
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|== | 2%
|
|=========================================================================================| 100%#==========================================================================================================
# Let's compare the performance of the two GBMs
#==========================================================================================================
gbm_perf1 <- h2o.performance(model = gbm_fit1,
newdata = test)
gbm_perf2 <- h2o.performance(model = gbm_fit2,
newdata = test)
gbm_perf3 <- h2o.performance(model = gbm_fit3,
newdata = test)#==========================================================================================================
# Print model performance
#==========================================================================================================
gbm_perf1H2OBinomialMetrics: gbm
MSE: 0.2398655
RMSE: 0.4897607
LogLoss: 0.8364659
Mean Per-Class Error: 0.333725
AUC: 0.6169213
Gini: 0.2338425
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 30 7 0.189189 =7/37
Yes 11 12 0.478261 =11/23
Totals 41 19 0.300000 =18/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.314972 0.571429 18
2 max f2 0.013415 0.761589 58
3 max f0point5 0.738993 0.636364 7
4 max accuracy 0.738993 0.716667 7
5 max precision 0.938118 1.000000 0
6 max recall 0.013415 1.000000 58
7 max specificity 0.938118 1.000000 0
8 max absolute_mcc 0.738993 0.396644 7
9 max min_per_class_accuracy 0.152995 0.608696 27
10 max mean_per_class_accuracy 0.462635 0.671563 15
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`gbm_perf2H2OBinomialMetrics: gbm
MSE: 0.3094912
RMSE: 0.5563193
LogLoss: 1.784196
Mean Per-Class Error: 0.4324324
AUC: 0.626322
Gini: 0.2526439
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 5 32 0.864865 =32/37
Yes 0 23 0.000000 =0/23
Totals 5 55 0.533333 =32/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.000046 0.589744 54
2 max f2 0.000046 0.782313 54
3 max f0point5 0.872488 0.636364 7
4 max accuracy 0.872488 0.716667 7
5 max precision 0.999979 1.000000 0
6 max recall 0.000046 1.000000 54
7 max specificity 0.999979 1.000000 0
8 max absolute_mcc 0.872488 0.396644 7
9 max min_per_class_accuracy 0.008380 0.565217 28
10 max mean_per_class_accuracy 0.872488 0.638660 7
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`gbm_perf3H2OBinomialMetrics: gbm
MSE: 0.2238611
RMSE: 0.4731396
LogLoss: 0.7295884
Mean Per-Class Error: 0.306698
AUC: 0.6333725
Gini: 0.266745
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 32 5 0.135135 =5/37
Yes 11 12 0.478261 =11/23
Totals 43 17 0.266667 =16/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.437536 0.600000 16
2 max f2 0.026584 0.756579 59
3 max f0point5 0.489286 0.666667 12
4 max accuracy 0.489286 0.733333 12
5 max precision 0.917153 1.000000 0
6 max recall 0.026584 1.000000 59
7 max specificity 0.917153 1.000000 0
8 max absolute_mcc 0.489286 0.417428 12
9 max min_per_class_accuracy 0.211870 0.608696 24
10 max mean_per_class_accuracy 0.437536 0.693302 16
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`#==========================================================================================================
# Retreive test set AUC
#==========================================================================================================
h2o.auc(gbm_perf1) [1] 0.6169213h2o.auc(gbm_perf2) [1] 0.626322h2o.auc(gbm_perf3) [1] 0.6333725To examine the scoring history, use the scoring_history method on a trained model.
If score_tree_interval is not specified, it will score at various intervals, as we can see for h2o.scoreHistory() below. However, regular 5-tree intervals are used for h2o.scoreHistory().
The gbm_fit2 was trained only using a training set (no validation set), so the scoring history is calculated for training set performance metrics only.
#==========================================================================================================
# To examine the scoring history, use the `scoring_history` method on a trained model.
# If `score_tree_interval` is not specified, it will score at various intervals, as we can
# see for `h2o.scoreHistory()` below. However, regular 5-tree intervals are used
# for `h2o.scoreHistory()`.
# The `gbm_fit2` was trained only using a training set (no validation set), so the scoring
# history is calculated for training set performance metrics only.
#==========================================================================================================
h2o.scoreHistory(gbm_fit2)Scoring History:
timestamp duration number_of_trees training_rmse training_logloss training_auc
1 2017-10-08 01:27:12 0.001 sec 0 0.47447 0.64254 0.50000
2 2017-10-08 01:27:12 0.009 sec 1 0.45247 0.59776 0.90156
3 2017-10-08 01:27:12 0.015 sec 2 0.43398 0.56188 0.90831
4 2017-10-08 01:27:12 0.021 sec 3 0.41807 0.53192 0.91432
5 2017-10-08 01:27:12 0.026 sec 4 0.40453 0.50680 0.91737
training_lift training_classification_error
1 1.00000 0.65772
2 2.92157 0.17785
3 2.92157 0.17114
4 2.92157 0.16443
5 2.92157 0.15436
---
timestamp duration number_of_trees training_rmse training_logloss training_auc
167 2017-10-08 01:27:16 3.849 sec 166 0.15266 0.09869 0.99797
168 2017-10-08 01:27:16 3.882 sec 167 0.15216 0.09817 0.99792
169 2017-10-08 01:27:16 3.916 sec 168 0.15174 0.09770 0.99792
170 2017-10-08 01:27:16 3.951 sec 169 0.15121 0.09716 0.99797
171 2017-10-08 01:27:16 3.991 sec 170 0.15070 0.09654 0.99802
172 2017-10-08 01:27:17 4.863 sec 500 0.10981 0.04117 0.99877
training_lift training_classification_error
167 2.92157 0.02685
168 2.92157 0.02685
169 2.92157 0.02685
170 2.92157 0.02685
171 2.92157 0.02685
172 2.92157 0.02349#==========================================================================================================
# When early stopping is used, we see that training stopped at 105 trees instead of the full 500.
# Since we used a validation set in `gbm_fit3`, both training and validation performance metrics
# are stored in the scoring history object. Take a look at the validation AUC to observe that the
# correct stopping tolerance was enforced.
#==========================================================================================================
h2o.scoreHistory(gbm_fit3)Scoring History:
timestamp duration number_of_trees training_rmse training_logloss training_auc
1 2017-10-08 01:27:18 0.001 sec 0 0.47447 0.64254 0.50000
2 2017-10-08 01:27:18 0.021 sec 5 0.39295 0.48547 0.92099
3 2017-10-08 01:27:18 0.045 sec 10 0.35296 0.41031 0.93035
4 2017-10-08 01:27:18 0.069 sec 15 0.33131 0.36613 0.93705
5 2017-10-08 01:27:18 0.090 sec 20 0.31451 0.33344 0.94768
6 2017-10-08 01:27:18 0.115 sec 25 0.29868 0.30512 0.95521
7 2017-10-08 01:27:18 0.137 sec 30 0.28750 0.28517 0.96141
training_lift training_classification_error validation_rmse validation_logloss validation_auc
1 1.00000 0.65772 0.48646 0.66638 0.50000
2 2.92157 0.15436 0.45331 0.59915 0.72439
3 2.92157 0.13758 0.46625 0.62275 0.70537
4 2.92157 0.12416 0.48076 0.65590 0.67707
5 2.92157 0.11074 0.48574 0.67175 0.67122
6 2.92157 0.09732 0.49773 0.70904 0.66829
7 2.92157 0.09060 0.50043 0.72274 0.68098
validation_lift validation_classification_error
1 1.00000 0.62121
2 1.32000 0.42424
3 1.32000 0.42424
4 0.00000 0.39394
5 2.64000 0.39394
6 2.64000 0.36364
7 2.64000 0.40909#==========================================================================================================
# Look at scoring history for third GBM model
#==========================================================================================================
plot(gbm_fit3,
timestep = "number_of_trees",
metric = "AUC")
plot(gbm_fit3,
timestep = "number_of_trees",
metric = "logloss")
There is overfitting for number of trees greater than 5.This can be observed from the graph above which shows the training error continues to decrease but the validation error starts to increase after 5 trees.We will arrrive at a better model by choosing number of trees to be less than 5.
#==========================================================================================================
# 4. Deep Learning
# H2O's Deep Learning algorithm is a multilayer feed-forward artificial neural network.
# It can also be used to train an autoencoder. In this example we will train
# a standard supervised prediction model.
# Train a default DL
# First we will train a basic DL model with default parameters. The DL model will infer the response
# distribution from the response encoding if it is not specified explicitly through the `distribution`
# argument. H2O's DL will not be reproducible if it is run on more than a single core, so in this example,
# the performance metrics below may vary slightly from what you see on your machine.
# In H2O's DL, early stopping is enabled by default, so below, it will use the training set and
# default stopping parameters to perform early stopping.
#==========================================================================================================
dl_fit1 <- h2o.deeplearning(x = x,
y = y,
training_frame = train,
model_id = "dl_fit1",
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|======================================================================= | 80%
|
|=========================================================================================| 100%#==========================================================================================================
# Train a DL with new architecture and more epochs.
# Next we will increase the number of epochs used in the GBM by setting `epochs=20` (the default is 10).
# Increasing the number of epochs in a deep neural net may increase performance of the model, however,
# you have to be careful not to overfit your model to your training data. To automatically find the optimal number of epochs,
# you must use H2O's early stopping functionality. Unlike the rest of the H2O algorithms, H2O's DL will
# use early stopping by default, so for comparison we will first turn off early stopping. We do this in the next example
# by setting `stopping_rounds=0`.
#==========================================================================================================
dl_fit2 <- h2o.deeplearning(x = x,
y = y,
training_frame = train,
model_id = "dl_fit2",
#validation_frame = valid, #only used if stopping_rounds > 0
epochs = 20,
hidden= c(10,10),
stopping_rounds = 0, # disable early stopping
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|================== | 20%
|
|=========================================================================================| 100%#==========================================================================================================
# Train a DL with early stopping
# This example will use the same model parameters as `dl_fit2`. This time, we will turn on
# early stopping and specify the stopping criterion. We will also pass a validation set, as is
# recommended for early stopping.
#==========================================================================================================
dl_fit3 <- h2o.deeplearning(x = x,
y = y,
training_frame = train,
model_id = "dl_fit3",
validation_frame = valid, #in DL, early stopping is on by default
epochs = 2,
hidden = c(10,10),
score_interval = 1, #used for early stopping
stopping_rounds = 3, #used for early stopping
stopping_metric = "AUC", #used for early stopping
stopping_tolerance = 0.0005, #used for early stopping
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|=========================================================================================| 100%#==========================================================================================================
# Let's compare the performance of the three DL models
#==========================================================================================================
dl_perf1 <- h2o.performance(model = dl_fit1,
newdata = test)
dl_perf2 <- h2o.performance(model = dl_fit2,
newdata = test)
dl_perf3 <- h2o.performance(model = dl_fit3,
newdata = test)
#==========================================================================================================
# Print model performance
#==========================================================================================================
dl_perf1H2OBinomialMetrics: deeplearning
MSE: 0.2413914
RMSE: 0.491316
LogLoss: 0.9163704
Mean Per-Class Error: 0.2849589
AUC: 0.6768508
Gini: 0.3537015
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 32 5 0.135135 =5/37
Yes 10 13 0.434783 =10/23
Totals 42 18 0.250000 =15/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.151863 0.634146 17
2 max f2 0.007122 0.777027 55
3 max f0point5 0.288452 0.746269 10
4 max accuracy 0.288452 0.766667 10
5 max precision 0.783162 1.000000 0
6 max recall 0.007122 1.000000 55
7 max specificity 0.783162 1.000000 0
8 max absolute_mcc 0.288452 0.512354 10
9 max min_per_class_accuracy 0.070772 0.594595 28
10 max mean_per_class_accuracy 0.174827 0.720329 14
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`dl_perf2H2OBinomialMetrics: deeplearning
MSE: 0.2209325
RMSE: 0.4700346
LogLoss: 0.7261213
Mean Per-Class Error: 0.2925969
AUC: 0.7003525
Gini: 0.4007051
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 25 12 0.324324 =12/37
Yes 6 17 0.260870 =6/23
Totals 31 29 0.300000 =18/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.136734 0.653846 28
2 max f2 0.016174 0.761589 58
3 max f0point5 0.364614 0.704225 11
4 max accuracy 0.364614 0.750000 11
5 max precision 0.863757 1.000000 0
6 max recall 0.016174 1.000000 58
7 max specificity 0.863757 1.000000 0
8 max absolute_mcc 0.364614 0.462774 11
9 max min_per_class_accuracy 0.153054 0.675676 27
10 max mean_per_class_accuracy 0.136734 0.707403 28
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`dl_perf3H2OBinomialMetrics: deeplearning
MSE: 0.2079788
RMSE: 0.4560469
LogLoss: 0.6326183
Mean Per-Class Error: 0.2820212
AUC: 0.7226792
Gini: 0.4453584
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 29 8 0.216216 =8/37
Yes 8 15 0.347826 =8/23
Totals 37 23 0.266667 =16/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.280369 0.652174 22
2 max f2 0.045496 0.761589 58
3 max f0point5 0.447421 0.666667 12
4 max accuracy 0.447421 0.733333 12
5 max precision 0.868179 1.000000 0
6 max recall 0.045496 1.000000 58
7 max specificity 0.868179 1.000000 0
8 max absolute_mcc 0.280369 0.435958 22
9 max min_per_class_accuracy 0.280369 0.652174 22
10 max mean_per_class_accuracy 0.280369 0.717979 22
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`#==========================================================================================================
# Retreive test set AUC
#==========================================================================================================
h2o.auc(dl_perf1) [1] 0.6768508h2o.auc(dl_perf2) [1] 0.7003525h2o.auc(dl_perf3) [1] 0.7226792#==========================================================================================================
# Scoring history
#==========================================================================================================
h2o.scoreHistory(dl_fit3)Scoring History:
timestamp duration training_speed epochs iterations samples training_rmse
1 2017-10-08 01:27:25 0.000 sec 0.00000 0 0.000000
2 2017-10-08 01:27:25 0.018 sec 21333 obs/sec 0.21477 1 64.000000 0.51330
3 2017-10-08 01:27:25 0.039 sec 32736 obs/sec 2.08725 10 622.000000 0.40466
training_logloss training_auc training_lift training_classification_error validation_rmse
1
2 0.87313 0.56403 1.94771 0.65772 0.55709
3 0.49752 0.82768 2.92157 0.26174 0.43714
validation_logloss validation_auc validation_lift validation_classification_error
1
2 1.00843 0.52488 0.00000 0.60606
3 0.59696 0.72195 2.64000 0.21212#==========================================================================================================
# confusion matrix
#==========================================================================================================
h2o.confusionMatrix(dl_fit3)Confusion Matrix (vertical: actual; across: predicted) for max f1 @ threshold = 0.155333974647658:
No Yes Error Rate
No 133 63 0.321429 =63/196
Yes 15 87 0.147059 =15/102
Totals 148 150 0.261745 =78/298#==========================================================================================================
# model diagnostics
#==========================================================================================================
plot(dl_fit3,
timestep = "epochs",
metric = "classification_error")
h2o.scoreHistory(dl_fit3)$epochs[1] 0.0000000 0.2147651 2.0872483h2o.scoreHistory(dl_fit3)$validation_classification_error[1] NaN 0.6060606 0.2121212#==========================================================================================================
# Look at scoring history for third DL model
#==========================================================================================================
# The model starts to overfitt as epoch goes beyond 2. The training error continues to decrease whereas the
# test error begins to increase.
plot(dl_fit3,
timestep = "epochs",
metric = "AUC")
#==========================================================================================================
# # Get the CV models from the `dl_fit3` object for third DL model
#==========================================================================================================
dl_fit3 <- h2o.deeplearning(x = x,
y = y,
training_frame = train,
model_id = "dl_fit3",
validation_frame = valid, #in DL, early stopping is on by default
epochs = 2,
nfolds = 3,
stopping_metric = "misclassification", #used for early stopping
hidden = c(10,10),
score_interval = 1, #used for early stopping
stopping_rounds = 5, #used for early stopping
#stopping_metric = "AUC", #used for early stopping
stopping_tolerance = 0.0005, #used for early stopping
seed = 1)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|====================================================================== | 79%
|
|=========================================================================================| 100%cv_models <- sapply(dl_fit3@model$cross_validation_models,
function(i) h2o.getModel(i$name))
# Plot the scoring history over time
plot(cv_models[[2]],
timestep = "epochs",
metric = "classification_error")
plot(dl_fit3,
timestep = "epochs",
metric = "classification_error")
cv_models[[1]]Model Details:
==============
H2OBinomialModel: deeplearning
Model ID: dl_fit3_cv_1
Status of Neuron Layers: predicting Hypertension, 2-class classification, bernoulli distribution, CrossEntropy loss, 862 weights/biases, 17.6 KB, 388 training samples, mini-batch size 1
layer units type dropout l1 l2 mean_rate rate_rms momentum mean_weight
1 1 72 Input 0.00 %
2 2 10 Rectifier 0.00 % 0.000000 0.000000 0.312233 0.455086 0.000000 0.002696
3 3 10 Rectifier 0.00 % 0.000000 0.000000 0.003896 0.004367 0.000000 -0.010156
4 4 2 Softmax 0.000000 0.000000 0.002739 0.001932 0.000000 0.597045
weight_rms mean_bias bias_rms
1
2 0.161645 0.504851 0.057905
3 0.299093 0.996581 0.031065
4 1.652194 -0.000000 0.018216
H2OBinomialMetrics: deeplearning
** Reported on training data. **
** Metrics reported on temporary training frame with 298 samples **
MSE: 0.1505877
RMSE: 0.3880564
LogLoss: 0.4643866
Mean Per-Class Error: 0.220613
AUC: 0.8475962
Gini: 0.6951923
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 105 23 0.179688 =23/128
Yes 17 48 0.261538 =17/65
Totals 122 71 0.207254 =40/193
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.293097 0.705882 69
2 max f2 0.116315 0.811170 114
3 max f0point5 0.563696 0.727700 36
4 max accuracy 0.499190 0.797927 45
5 max precision 0.897045 1.000000 0
6 max recall 0.014746 1.000000 178
7 max specificity 0.897045 1.000000 0
8 max absolute_mcc 0.335698 0.549399 64
9 max min_per_class_accuracy 0.261208 0.765625 78
10 max mean_per_class_accuracy 0.293097 0.779387 69
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`
H2OBinomialMetrics: deeplearning
** Reported on validation data. **
** Metrics reported on full validation frame **
MSE: 0.2064037
RMSE: 0.4543167
LogLoss: 0.6735717
Mean Per-Class Error: 0.2724563
AUC: 0.7066773
Gini: 0.4133545
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 53 15 0.220588 =15/68
Yes 12 25 0.324324 =12/37
Totals 65 40 0.257143 =27/105
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.276746 0.649351 38
2 max f2 0.012913 0.755102 94
3 max f0point5 0.276746 0.634518 38
4 max accuracy 0.276746 0.742857 38
5 max precision 0.902949 1.000000 0
6 max recall 0.012913 1.000000 94
7 max specificity 0.902949 1.000000 0
8 max absolute_mcc 0.276746 0.447676 38
9 max min_per_class_accuracy 0.211203 0.702703 44
10 max mean_per_class_accuracy 0.276746 0.727544 38
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`As an alternative to manual tuning, or “hand tuning”, we can use the h2o.grid() function to perform either a Cartesian or Randon Grid Search (RGS). Random Grid Search is usually a quicker way to find a good model, so we will provide a example of how to use H2O’s Random Grid Search on a DNN. One handy feature of RGS is that you can specify how long you would like to execute the grid for – this can be based on a time, number of models, or a performance-metric-based stopping criterion. In the example below,we will train the DNN grid for 600 seconds (10 minutes). First define a grid of Deep Learning hyperparamters and specify the search_criteria .
#==========================================================================================================
# Deep Learning Grid Search third DL model
#==========================================================================================================
# As an alternative to manual tuning, or “hand tuning”, we can use the h2o.grid() function to perform either a
# Cartesian or Randon Grid Search (RGS). Random Grid Search is usually a quicker way to find a good model,
# so we will provide a example of how to use H2O’s Random Grid Search on a DNN.
# One handy feature of RGS is that you can specify how long you would like to execute the grid for – this can be
# based on a time, number of models, or a performance-metric-based stopping criterion. In the example below,
# we will train the DNN grid for 600 seconds (10 minutes).
# First define a grid of Deep Learning hyperparamters and specify the search_criteria .
activation_opt <- c("Rectifier", "Maxout", "Tanh")
l1_opt <- c(0, 0.00001, 0.0001, 0.001, 0.01)
l2_opt <- c(0, 0.00001, 0.0001, 0.001, 0.01)
hyper_params <- list(activation = activation_opt, l1 = l1_opt, l2 = l2_opt)
search_criteria <- list(strategy = "RandomDiscrete", max_runtime_secs = 600)
# Rather than comparing models by using cross-validation (which is “better” but takes longer), we will simply
# partition our training set into two pieces – one for training and one for validiation.
# This will split the train frame into an 80% and 20% partition of the rows.
splits <- h2o.splitFrame(train, ratios = 0.8, seed = 1)
#Train the random grid. Fixed non-default parameters such as hidden=c(20,20) can be passed directly to
#the h2o.grid() function.
dl_grid <- h2o.grid("deeplearning", x = x, y = y,
grid_id = "dl_grid",
training_frame = splits[[1]],
validation_frame = splits[[2]],
seed = 1,
hidden = c(20,20),
hyper_params = hyper_params,
search_criteria = search_criteria)
|
| | 0%
|
| | 1%
|
|= | 1%#Once we have trained the grid, we can collect the results and sort by our model performance metric of choice.
dl_gridperf <- h2o.getGrid(grid_id = "dl_grid",
sort_by = "accuracy",
decreasing = TRUE)
print(dl_gridperf)H2O Grid Details
================
Grid ID: dl_grid
Used hyper parameters:
- activation
- l1
- l2
Number of models: 75
Number of failed models: 0
Hyper-Parameter Search Summary: ordered by decreasing accuracy
activation l1 l2 model_ids accuracy
1 Maxout 1.0E-5 0.001 dl_grid_model_59 0.7818181818181819
2 Maxout 1.0E-5 0.0 dl_grid_model_44 0.7818181818181819
3 Maxout 0.001 0.001 dl_grid_model_19 0.7818181818181819
4 Maxout 0.01 0.0 dl_grid_model_43 0.7818181818181819
5 Rectifier 0.01 1.0E-4 dl_grid_model_64 0.7818181818181819
---
activation l1 l2 model_ids accuracy
70 Tanh 1.0E-4 1.0E-5 dl_grid_model_62 0.5272727272727273
71 Tanh 0.01 1.0E-4 dl_grid_model_35 0.5272727272727273
72 Tanh 0.0 1.0E-4 dl_grid_model_68 0.509090909090909
73 Tanh 1.0E-5 1.0E-4 dl_grid_model_56 0.509090909090909
74 Tanh 0.0 0.0 dl_grid_model_63 0.509090909090909
75 Tanh 0.001 0.0 dl_grid_model_38 0.49090909090909096#Grab the model_id for the top DL model, chosen by validation error.
best_dl_model_id <- dl_gridperf@model_ids[[1]]
best_dl <- h2o.getModel(best_dl_model_id)
#Now let’s evaluate the model performance on a test set so we get an honest estimate of top model
#performance.
best_dl_perf <- h2o.performance(model = best_dl, newdata = test)
h2o.mse(best_dl_perf)[1] 0.2421704#==========================================================================================================
# 5. Naive Bayes model
# The Naive Bayes (NB) algorithm does not usually beat an algorithm like a Random Forest
# or GBM, however it is still a popular algorithm, especially in the text domain (when your
# input is text encoded as "Bag of Words", for example). The Naive Bayes algorithm is for
# binary or multiclass classification problems only, not regression. Therefore, your response
# must be a factor instead of a numeric.
# First we will train a basic NB model with default parameters.
#==========================================================================================================
nb_fit1 <- h2o.naiveBayes(x = x,
y = y,
training_frame = train,
model_id = "nb_fit1")Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|=========================================================================================| 100%#==========================================================================================================
# Train a NB model with Laplace Smoothing
# One of the few tunable model parameters for the Naive Bayes algorithm is the amount of Laplace
# smoothing. The H2O Naive Bayes model will not use any Laplace smoothing by default.
#==========================================================================================================
nb_fit2 <- h2o.naiveBayes(x = x,
y = y,
training_frame = train,
model_id = "nb_fit2",
laplace = 6)Dropping bad and constant columns: [race_pacific].
|
| | 0%
|
|=========================================================================================| 100%#==========================================================================================================
# Let's compare the performance of the two NB models
#==========================================================================================================
nb_perf1 <- h2o.performance(model = nb_fit1,
newdata = test)
nb_perf2 <- h2o.performance(model = nb_fit2,
newdata = test)
#==========================================================================================================
# Print model performance
#==========================================================================================================
nb_perf1H2OBinomialMetrics: naivebayes
MSE: 0.1797905
RMSE: 0.4240172
LogLoss: 0.7532077
Mean Per-Class Error: 0.2226792
AUC: 0.7121034
Gini: 0.4242068
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 35 2 0.054054 =2/37
Yes 9 14 0.391304 =9/23
Totals 44 16 0.183333 =11/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.608819 0.717949 15
2 max f2 0.003638 0.761589 58
3 max f0point5 0.608819 0.804598 15
4 max accuracy 0.608819 0.816667 15
5 max precision 0.999242 1.000000 0
6 max recall 0.003638 1.000000 58
7 max specificity 0.999242 1.000000 0
8 max absolute_mcc 0.608819 0.609805 15
9 max min_per_class_accuracy 0.313477 0.695652 22
10 max mean_per_class_accuracy 0.608819 0.777321 15
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`nb_perf2H2OBinomialMetrics: naivebayes
MSE: 0.1774692
RMSE: 0.4212709
LogLoss: 0.5977346
Mean Per-Class Error: 0.2361927
AUC: 0.7614571
Gini: 0.5229142
Confusion Matrix (vertical: actual; across: predicted) for F1-optimal threshold:
No Yes Error Rate
No 34 3 0.081081 =3/37
Yes 9 14 0.391304 =9/23
Totals 43 17 0.200000 =12/60
Maximum Metrics: Maximum metrics at their respective thresholds
metric threshold value idx
1 max f1 0.437234 0.700000 16
2 max f2 0.009638 0.756579 59
3 max f0point5 0.437234 0.769231 16
4 max accuracy 0.437234 0.800000 16
5 max precision 0.993291 1.000000 0
6 max recall 0.009638 1.000000 59
7 max specificity 0.993291 1.000000 0
8 max absolute_mcc 0.437234 0.569276 16
9 max min_per_class_accuracy 0.211192 0.702703 27
10 max mean_per_class_accuracy 0.437234 0.763807 16
Gains/Lift Table: Extract with `h2o.gainsLift(<model>, <data>)` or `h2o.gainsLift(<model>, valid=<T/F>, xval=<T/F>)`LS0tCnRpdGxlOiAiRGVlcCBMZWFybmluZyB3aXRoIGgyMCIKb3V0cHV0OiBodG1sX25vdGVib29rCmF1dGhvcjogTmFuYSBCb2F0ZW5nCmRmX3ByaW50OiBwYWdlZApUaW1lOiAnYHIgU3lzLnRpbWUoKWAnCmRhdGU6ICJgciBmb3JtYXQoU3lzLnRpbWUoKSwgJyVCICVkLCAlWScpYCIKLS0tCgoKCmBgYHtyIHNldHVwLCBpbmNsdWRlPUZBTFNFfQprbml0cjo6b3B0c19jaHVuayRzZXQoY2FjaGU9VFJVRSkKYGBgCgoKVGhlIGRhdGEgY29uc2lzdCBpZiBzdXJ2ZXlzIGFkbWluaXN0ZXJlZCB0byBwZW9wbGUgd2hvIGxpdmUgbmVhciBhIGNvbW11bml0eSBoZWFsdGggZmFjaWx0eS4gVGhlIGdvYWwgaGVyZSBpcyB0byBwcmVkaWN0IHRoZSByYXRlIG9mIGh5cGVydGVuc2lvbiBhbW9uZyB0aGUgcmVzaWRlbnRzIGdpdmVuIHByZWRpY3RvcnMgc3VjaCBhcyBhZ2UscmFjZSxnZW5kZXIsZWR1Y2F0aW9uIGFuZCBlbXBsb3ltZW50IHN0YXR1cy4KCmBgYHtyfQpybShsaXN0ID1jKCkpCgoKbGlicmFyeShoMm8pCmxpYnJhcnkoaDJvRW5zZW1ibGUpCmxpYnJhcnkodGlkeXZlcnNlKQoKaDJvLmluaXQobnRocmVhZHMgPSAtMSwgI051bWJlciBvZiB0aHJlYWRzIC0xIG1lYW5zIHVzZSBhbGwgY29yZXMgb24geW91ciBtYWNoaW5lCiAgICAgICAgIG1heF9tZW1fc2l6ZSA9ICI4RyIpICAjbWF4IG1lbSBzaXplIGlzIHRoZSBtYXhpbXVtIG1lbW9yeSB0byBhbGxvY2F0ZSB0byBIMk8KCgpsb2NhbEgyTyA9IGgyby5pbml0KGlwID0gJ2xvY2FsaG9zdCcsIHBvcnQgPSA1NDMyMSwgbnRocmVhZHMgPSAtMSxtYXhfbWVtX3NpemUgPSAiOEciKQoKCmxvYW5fY3N2IDwtICIvVXNlcnMvbmFuYWFrd2FzaWFiYXlpZWJvYXRlbmcvRG9jdW1lbnRzL21lbXBoaXNjbGFzc2VzYm9va3MvRGF0YU1pbmluZ3NjaWVuY2UvVUlDUHJvamVjdC9Xb3JrYm9vazMuY3N2IgpkYXRhY2hpY2FnbyA8LSBoMm8uaW1wb3J0RmlsZShsb2FuX2NzdikgIApkaW0oZGF0YWNoaWNhZ28pCgpoZWFkKGRhdGFjaGljYWdvKQoKaDJvLm5hbWVzKChkYXRhY2hpY2FnbykpCgpzdHIoZGF0YWNoaWNhZ28pCgoKYGBgCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIExvb2sgYXQgIHRoZSAgc3RydWN0dXJlIG9mIHRoZSBkYXRhIHdpdGggdGhlIGdsaW1wc2UgZnVuY3Rpb24gaW4gCiMgIGRwbHlyICBwYWNrYWdlCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgpzdHIoZGF0YSkKCmRwbHlyOjpnbGltcHNlKGRhdGEpCgpzdW1tYXJ5KGRhdGEpCgoKYGBgCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KI2NoZWNrIHRoZSBudW1iZXIgb2YgbWlzc2luZyByb3dzCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09Cgpjb2xTdW1zKGlzLm5hLmRhdGEuZnJhbWUoZGF0YSkpCgoKZGF0YVshY29tcGxldGUuY2FzZXMoZGF0YSksXSU+JWhlYWQoKQoKCmRhdGFbd2hpY2goZGF0YSRBc2lhbl9hbmNlc3RyeSE9Ii4iKSxdCgoKZGF0YSRBc2lhbl9hbmNlc3RyeT1pZmVsc2UoZGF0YSRBc2lhbl9hbmNlc3RyeT09Ii4iLCI8TkE+IixkYXRhJEFzaWFuX2FuY2VzdHJ5KQoKCmRhdGE9ZGF0YVtjb21wbGV0ZS5jYXNlcyhkYXRhKSxdCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojTk8gQ09ERUQgUkVTUE9OU0UgQVBQTElDQUJMRSAoU1BFQ0lGWSkKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiNkYXRhJT4lZHBseXI6OmZpbHRlcihzdHJfZGV0ZWN0KEh5cGVydGVuc2lvbiwgIk5PIENPREVEIFJFU1BPTlNFIEFQUExJQ0FCTEUgKFNQRUNJRlkpIikpCgpuZGF0YT1kYXRhJT4lZHBseXI6OnNlbGVjdCgtSUQpJT4lZHBseXI6OmZpbHRlcihIeXBlcnRlbnNpb24hPSJOTyBDT0RFRCBSRVNQT05TRSBBUFBMSUNBQkxFIChTUEVDSUZZKSIsRW1wbG95bWVudAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAhPSJOTyBDT0RFRCBSRVNQT05TRSBBUFBMSUNBQkxFIChMRUFWRSBOT1RFIEZJUlNUKSIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLEhpc3BhbmljIT0iTk8gQ09ERUQgUkVTUE9OU0UgQVBQTElDQUJMRSAoU1BFQ0lGWSkiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBFZHVjYXRpb24hPSJOTyBDT0RFRCBSRVNQT05TRSBBUFBMSUNBQkxFIChTUEVDSUZZKSIpCgoKCgoKbmRhdGE9bXV0YXRlX2lmKG5kYXRhLGlzLmNoYXJhY3Rlcixhcy5mYWN0b3IpCgpzdHIobmRhdGEpCmBgYAoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIGltcG9ydCBSIG9iamVjdCB0byB0aGUgSDJPIGNsb3VkLgojY29udmVydCByIGRhdGEgdG8gaDJvIG9iamVjdAojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKZGF0YWgyMD1hcy5oMm8obmRhdGEpCgpzdHIoZGF0YWgyMCkKCmBgYAoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIFBhcnRpdGlvbiB0aGUgZGF0YSBpbnRvIHRyYWluaW5nLCB2YWxpZGF0aW9uIGFuZCB0ZXN0IHNldHMKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCnNwbGl0cyA8LSBoMm8uc3BsaXRGcmFtZShkYXRhID0gZGF0YWgyMCwgCiAgICAgICAgICAgICAgICAgICAgICAgICByYXRpb3MgPSBjKDAuNywgMC4xNSksICAjcGFydGl0aW9uIGRhdGEgaW50byA3MCUsIDE1JSwgMTUlIGNodW5rcwogICAgICAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpICAjc2V0dGluZyBhIHNlZWQgd2lsbCBndWFyYW50ZWUgcmVwcm9kdWNpYmlsaXR5CnRyYWluIDwtIHNwbGl0c1tbMV1dCnZhbGlkIDwtIHNwbGl0c1tbMl1dCnRlc3QgPC0gc3BsaXRzW1szXV0KCgoKIyBJZGVudGlmeSByZXNwb25zZSBhbmQgcHJlZGljdG9yIHZhcmlhYmxlcwp5IDwtICJIeXBlcnRlbnNpb24iCnggPC0gc2V0ZGlmZihuYW1lcyhkYXRhaDIwKSwgeSkgIApgYGAKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiNnbG0vbG9naXN0aWMKI3NpbWlsYXIgdG8gUidzIGdsbSwgaDJvLmdsbSBoYXMgdGhlIGZhbWlseSBhcmd1bWVudAojIDEuIExldCdzIHN0YXJ0IHdpdGggYSBiYXNpYyBiaW5vbWlhbCBHZW5lcmFsaXplZCBMaW5lYXIgTW9kZWwKIyBCeSBkZWZhdWx0LCBoMm8uZ2xtIHVzZXMgYSByZWd1bGFyaXplZCwgZWxhc3RpYyBuZXQgbW9kZWwKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpnbG1fZml0MSA8LSBoMm8uZ2xtKHggPSB4LCAKICAgICAgICAgICAgICAgICAgICB5ID0geSwgCiAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICBtb2RlbF9pZCA9ICJnbG1fZml0MSIsCiAgICAgICAgICAgICAgICAgICAgZmFtaWx5ID0gImJpbm9taWFsIikgCgpgYGAKCgoKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBOZXh0IHdlIHdpbGwgZG8gc29tZSBhdXRvbWF0aWMgdHVuaW5nIGJ5IHBhc3NpbmcgaW4gYSB2YWxpZGF0aW9uIGZyYW1lIGFuZCBzZXR0aW5nIAojIGBsYW1iZGFfc2VhcmNoID0gVHJ1ZWAuICBTaW5jZSB3ZSBhcmUgdHJhaW5pbmcgYSBHTE0gd2l0aCByZWd1bGFyaXphdGlvbiwgd2Ugc2hvdWxkIAojIHRyeSB0byBmaW5kIHRoZSByaWdodCBhbW91bnQgb2YgcmVndWxhcml6YXRpb24gKHRvIGF2b2lkIG92ZXJmaXR0aW5nKS4gIFRoZSBtb2RlbCAKIyBwYXJhbWV0ZXIsIGBsYW1iZGFgLCBjb250cm9scyB0aGUgYW1vdW50IG9mIHJlZ3VsYXJpemF0aW9uIGluIGEgR0xNIG1vZGVsIGFuZCB3ZSBjYW4gCiMgZmluZCB0aGUgb3B0aW1hbCB2YWx1ZSBmb3IgYGxhbWJkYWAgYXV0b21hdGljYWxseSBieSBzZXR0aW5nIGBsYW1iZGFfc2VhcmNoID0gVFJVRWAgCiMgYW5kIHBhc3NpbmcgaW4gYSB2YWxpZGF0aW9uIGZyYW1lICh3aGljaCBpcyB1c2VkIHRvIGV2YWx1YXRlIG1vZGVsIHBlcmZvcm1hbmNlIHVzaW5nIGEgCiMgcGFydGljdWxhciB2YWx1ZSBvZiBsYW1iZGEpLgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmdsbV9maXQyIDwtIGgyby5nbG0oeCA9IHgsIAogICAgICAgICAgICAgICAgICAgIHkgPSB5LCAKICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gImdsbV9maXQyIiwKICAgICAgICAgICAgICAgICAgICB2YWxpZGF0aW9uX2ZyYW1lID0gdmFsaWQsCiAgICAgICAgICAgICAgICAgICAgZmFtaWx5ID0gImJpbm9taWFsIiwKICAgICAgICAgICAgICAgICAgICBsYW1iZGFfc2VhcmNoID0gVFJVRSkKCmBgYAoKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiMgTGV0J3MgY29tcGFyZSB0aGUgcGVyZm9ybWFuY2Ugb2YgdGhlIHR3byBHTE1zCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKZ2xtX3BlcmYxIDwtIGgyby5wZXJmb3JtYW5jZShtb2RlbCA9IGdsbV9maXQxLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSB0ZXN0KQpnbG1fcGVyZjIgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gZ2xtX2ZpdDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbmV3ZGF0YSA9IHRlc3QpCgoKCihnbG1fcGVyZjEpICAKKGdsbV9wZXJmMikgCgoKYGBgCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIEluc3RlYWQgb2YgcHJpbnRpbmcgdGhlIGVudGlyZSBtb2RlbCBwZXJmb3JtYW5jZSBtZXRyaWNzIG9iamVjdCwgCiMgaXQgaXMgcHJvYmFibHkgZWFzaWVyIHRvIHByaW50IGp1c3QgdGhlIG1ldHJpYyB0aGF0IHlvdSBhcmUgaW50ZXJlc3RlZCBpbiBjb21wYXJpbmcuCiMgUmV0cmVpdmUgdGVzdCBzZXQgQVVDCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKaDJvLmF1YyhnbG1fcGVyZjEpICAKaDJvLmF1YyhnbG1fcGVyZjIpICAKCmBgYAoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBDb21wYXJlIHRlc3QgQVVDIHRvIHRoZSB0cmFpbmluZyBBVUMgYW5kIHZhbGlkYXRpb24gQVVDCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKaDJvLmF1YyhnbG1fZml0MiwgdHJhaW4gPSBUUlVFKSAgCmgyby5hdWMoZ2xtX2ZpdDIsIHZhbGlkID0gVFJVRSkgCgpnbG1fZml0MkBtb2RlbCR2YWxpZGF0aW9uX21ldHJpY3MgIAoKCgpgYGAKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiMgMi4gUmFuZG9tIEZvcmVzdAojIEgyTydzIFJhbmRvbSBGb3Jlc3QgKFJGKSBpbXBsZW1lbnRzIGEgZGlzdHJpYnV0ZWQgdmVyc2lvbiBvZiB0aGUgc3RhbmRhcmQgCiMgUmFuZG9tIEZvcmVzdCBhbGdvcml0aG0gYW5kIHZhcmlhYmxlIGltcG9ydGFuY2UgbWVhc3VyZXMuCiMgRmlyc3Qgd2Ugd2lsbCB0cmFpbiBhIGJhc2ljIFJhbmRvbSBGb3Jlc3QgbW9kZWwgd2l0aCBkZWZhdWx0IHBhcmFtZXRlcnMuIAojIFRoZSBSYW5kb20gRm9yZXN0IG1vZGVsIHdpbGwgaW5mZXIgdGhlIHJlc3BvbnNlIGRpc3RyaWJ1dGlvbiBmcm9tIHRoZSByZXNwb25zZSBlbmNvZGluZy4gCiMgQSBzZWVkIGlzIHJlcXVpcmVkIGZvciByZXByb2R1Y2liaWxpdHkuCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKcmZfZml0MSA8LSBoMm8ucmFuZG9tRm9yZXN0KHggPSB4LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgeSA9IHksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWxfaWQgPSAicmZfZml0MSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZWVkID0gMSkKCgpgYGAKCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIE5leHQgd2Ugd2lsbCBpbmNyZWFzZSB0aGUgbnVtYmVyIG9mIHRyZWVzIHVzZWQgaW4gdGhlIGZvcmVzdCBieSBzZXR0aW5nIGBudHJlZXMgPSAxMDBgLiAgCiMgVGhlIGRlZmF1bHQgbnVtYmVyIG9mIHRyZWVzIGluIGFuIEgyTyBSYW5kb20gRm9yZXN0IGlzIDUwLCBzbyB0aGlzIFJGIHdpbGwgYmUgdHdpY2UgYXMgCiMgYmlnIGFzIHRoZSBkZWZhdWx0LiAgVXN1YWxseSBpbmNyZWFzaW5nIHRoZSBudW1iZXIgb2YgdHJlZXMgaW4gYSBSRiB3aWxsIGluY3JlYXNlIAojIHBlcmZvcm1hbmNlIGFzIHdlbGwuICBVbmxpa2UgR3JhZGllbnQgQm9vc3RpbmcgTWFjaGluZXMgKEdCTXMpLCBSYW5kb20gRm9yZXN0cyBhcmUgZmFpcmx5IAojIHJlc2lzdGFudCAoYWx0aG91Z2ggbm90IGZyZWUgZnJvbSkgb3ZlcmZpdHRpbmcuCiMgU2VlIHRoZSBHQk0gZXhhbXBsZSBiZWxvdyBmb3IgYWRkaXRpb25hbCBndWlkYW5jZSBvbiBwcmV2ZW50aW5nIG92ZXJmaXR0aW5nIHVzaW5nIEgyTydzIAojIGVhcmx5IHN0b3BwaW5nIGZ1bmN0aW9uYWxpdHkuCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKCnJmX2ZpdDIgPC0gaDJvLnJhbmRvbUZvcmVzdCh4ID0geCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHkgPSB5LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gInJmX2ZpdDIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgI3ZhbGlkYXRpb25fZnJhbWUgPSB2YWxpZCwgICNvbmx5IHVzZWQgaWYgc3RvcHBpbmdfcm91bmRzID4gMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbnRyZWVzID0gMTAwLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpCgoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBMZXQncyBjb21wYXJlIHRoZSBwZXJmb3JtYW5jZSBvZiB0aGUgdHdvIFJGcwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCnJmX3BlcmYxIDwtIGgyby5wZXJmb3JtYW5jZShtb2RlbCA9IHJmX2ZpdDEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXdkYXRhID0gdGVzdCkKcmZfcGVyZjIgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gcmZfZml0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSB0ZXN0KQoKYGBgCgoKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiMgUHJpbnQgbW9kZWwgcGVyZm9ybWFuY2UKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpyZl9wZXJmMQpyZl9wZXJmMgoKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CiMgUmV0cmVpdmUgdGVzdCBzZXQgQVVDCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKaDJvLmF1YyhyZl9wZXJmMSkgIApoMm8uYXVjKHJmX3BlcmYyKSAgCgoKYGBgCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBDcm9zcy12YWxpZGF0ZSBwZXJmb3JtYW5jZQojIFJhdGhlciB0aGFuIHVzaW5nIGhlbGQtb3V0IHRlc3Qgc2V0IHRvIGV2YWx1YXRlIG1vZGVsIHBlcmZvcm1hbmNlLCBhIHVzZXIgbWF5IHdpc2ggCiMgdG8gZXN0aW1hdGUgbW9kZWwgcGVyZm9ybWFuY2UgdXNpbmcgY3Jvc3MtdmFsaWRhdGlvbi4gVXNpbmcgdGhlIFJGIGFsZ29yaXRobSAKIyAod2l0aCBkZWZhdWx0IG1vZGVsIHBhcmFtZXRlcnMpIGFzIGFuIGV4YW1wbGUsIHdlIGRlbW9uc3RyYXRlIGhvdyB0byBwZXJmb3JtIGstZm9sZCAKIyBjcm9zcy12YWxpZGF0aW9uIHVzaW5nIEgyTy4gTm8gY3VzdG9tIGNvZGUgb3IgbG9vcHMgYXJlIHJlcXVpcmVkLCB5b3Ugc2ltcGx5IHNwZWNpZnkgCiMgdGhlIG51bWJlciBvZiBkZXNpcmVkIGZvbGRzIGluIHRoZSBuZm9sZHMgYXJndW1lbnQuCiMgU2luY2Ugd2UgYXJlIG5vdCBnb2luZyB0byB1c2UgYSB0ZXN0IHNldCBoZXJlLCB3ZSBjYW4gdXNlIHRoZSBvcmlnaW5hbCAoZnVsbCkgZGF0YXNldCwgCiMgd2hpY2ggd2UgY2FsbGVkIGRhdGEgcmF0aGVyIHRoYW4gdGhlIHN1YnNhbXBsZWQgYHRyYWluYCBkYXRhc2V0LiBOb3RlIHRoYXQgdGhpcyB3aWxsIAojIHRha2UgYXBwcm94aW1hdGVseSBrIChuZm9sZHMpIHRpbWVzIGxvbmdlciB0aGFuIHRyYWluaW5nIGEgc2luZ2xlIFJGIG1vZGVsLCBzaW5jZSBpdCAKIyB3aWxsIHRyYWluIGsgbW9kZWxzIGluIHRoZSBjcm9zcy12YWxpZGF0aW9uIHByb2Nlc3MgKHRyYWluZWQgb24gbihrLTEpL2sgcm93cyksIGluIAojIGFkZGl0aW9uIHRvIHRoZSBmaW5hbCBtb2RlbCB0cmFpbmVkIG9uIHRoZSBmdWxsIHRyYWluaW5nX2ZyYW1lIGRhdGFzZXQgd2l0aCBuIHJvd3MuCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCnJmX2ZpdDMgPC0gaDJvLnJhbmRvbUZvcmVzdCh4ID0geCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHkgPSB5LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gInJmX2ZpdDMiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZm9sZHMgPSA1KQoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBUbyBldmFsdWF0ZSB0aGUgY3Jvc3MtdmFsaWRhdGVkIEFVQywgZG8gdGhlIGZvbGxvd2luZzoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpoMm8uYXVjKHJmX2ZpdDMsIHh2YWwgPSBUUlVFKSAgCgoKYGBgCgoKCgoKYGBge3J9CgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIDMuIEdyYWRpZW50IEJvb3N0aW5nIE1hY2hpbmUKIyBIMk8ncyBHcmFkaWVudCBCb29zdGluZyBNYWNoaW5lIChHQk0pIG9mZmVycyBhIFN0b2NoYXN0aWMgR0JNLCB3aGljaCBjYW4gCiMgaW5jcmVhc2UgcGVyZm9ybWFuY2UgcXVpdGUgYSBiaXQgY29tcGFyZWQgdG8gdGhlIG9yaWdpbmFsIEdCTSBpbXBsZW1lbnRhdGlvbi4KCiMgTm93IHdlIHdpbGwgdHJhaW4gYSBiYXNpYyBHQk0gbW9kZWwKIyBUaGUgR0JNIG1vZGVsIHdpbGwgaW5mZXIgdGhlIHJlc3BvbnNlIGRpc3RyaWJ1dGlvbiBmcm9tIHRoZSByZXNwb25zZSBlbmNvZGluZyBpZiBub3Qgc3BlY2lmaWVkIAojIGV4cGxpY2l0bHkgdGhyb3VnaCB0aGUgYGRpc3RyaWJ1dGlvbmAgYXJndW1lbnQuIEEgc2VlZCBpcyByZXF1aXJlZCBmb3IgcmVwcm9kdWNpYmlsaXR5LgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmdibV9maXQxIDwtIGgyby5nYm0oeCA9IHgsCiAgICAgICAgICAgICAgICAgICAgeSA9IHksCiAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICBtb2RlbF9pZCA9ICJnYm1fZml0MSIsCiAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpCgoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBOZXh0IHdlIHdpbGwgaW5jcmVhc2UgdGhlIG51bWJlciBvZiB0cmVlcyB1c2VkIGluIHRoZSBHQk0gYnkgc2V0dGluZyBgbnRyZWVzPTUwMGAuICAKIyBUaGUgZGVmYXVsdCBudW1iZXIgb2YgdHJlZXMgaW4gYW4gSDJPIEdCTSBpcyA1MCwgc28gdGhpcyBHQk0gd2lsbCB0cmFpbmVkIHVzaW5nIHRlbiB0aW1lcyAKIyB0aGUgZGVmYXVsdC4gIEluY3JlYXNpbmcgdGhlIG51bWJlciBvZiB0cmVlcyBpbiBhIEdCTSBpcyBvbmUgd2F5IHRvIGluY3JlYXNlIHBlcmZvcm1hbmNlIAojIG9mIHRoZSBtb2RlbCwgaG93ZXZlciwgeW91IGhhdmUgdG8gYmUgY2FyZWZ1bCBub3QgdG8gb3ZlcmZpdCB5b3VyIG1vZGVsIHRvIHRoZSB0cmFpbmluZyBkYXRhIAojIGJ5IHVzaW5nIHRvbyBtYW55IHRyZWVzLiAgVG8gYXV0b21hdGljYWxseSBmaW5kIHRoZSBvcHRpbWFsIG51bWJlciBvZiB0cmVlcywgeW91IG11c3QgdXNlIAojIEgyTydzIGVhcmx5IHN0b3BwaW5nIGZ1bmN0aW9uYWxpdHkuICBUaGlzIGV4YW1wbGUgd2lsbCBub3QgZG8gdGhhdCwgaG93ZXZlciwgdGhlIGZvbGxvd2luZyAKIyBleGFtcGxlIHdpbGwuCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKZ2JtX2ZpdDIgPC0gaDJvLmdibSh4ID0geCwKICAgICAgICAgICAgICAgICAgICB5ID0geSwKICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gImdibV9maXQyIiwKICAgICAgICAgICAgICAgICAgICAjdmFsaWRhdGlvbl9mcmFtZSA9IHZhbGlkLCAgI29ubHkgdXNlZCBpZiBzdG9wcGluZ19yb3VuZHMgPiAwCiAgICAgICAgICAgICAgICAgICAgbnRyZWVzID0gNTAwLAogICAgICAgICAgICAgICAgICAgIHNlZWQgPSAxKQoKCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBXZSB3aWxsIGFnYWluIHNldCBgbnRyZWVzID0gNTAwYCwgaG93ZXZlciwgdGhpcyB0aW1lIHdlIHdpbGwgdXNlIGVhcmx5IHN0b3BwaW5nIGluIG9yZGVyIHRvIAojIHByZXZlbnQgb3ZlcmZpdHRpbmcgKGZyb20gdG9vIG1hbnkgdHJlZXMpLiAgQWxsIG9mIEgyTydzIGFsZ29yaXRobXMgaGF2ZSBlYXJseSBzdG9wcGluZyBhdmFpbGFibGUsIAojIGhvd2V2ZXIgZWFybHkgc3RvcHBpbmcgaXMgbm90IGVuYWJsZWQgYnkgZGVmYXVsdCAod2l0aCB0aGUgZXhjZXB0aW9uIG9mIERlZXAgTGVhcm5pbmcpLiAgCiMgVGhlcmUgYXJlIHNldmVyYWwgcGFyYW1ldGVycyB0aGF0IHNob3VsZCBiZSB1c2VkIHRvIGNvbnRyb2wgZWFybHkgc3RvcHBpbmcuICBUaGUgdGhyZWUgdGhhdCBhcmUgCiMgY29tbW9uIHRvIGFsbCB0aGUgYWxnb3JpdGhtcyBhcmU6IGBzdG9wcGluZ19yb3VuZHNgLCBgc3RvcHBpbmdfbWV0cmljYCBhbmQgYHN0b3BwaW5nX3RvbGVyYW5jZWAuICAKIyBUaGUgc3RvcHBpbmcgbWV0cmljIGlzIHRoZSBtZXRyaWMgYnkgd2hpY2ggeW91J2QgbGlrZSB0byBtZWFzdXJlIHBlcmZvcm1hbmNlLCBhbmQgc28gd2Ugd2lsbCBjaG9vc2UgCiMgQVVDIGhlcmUuICBUaGUgYHNjb3JlX3RyZWVfaW50ZXJ2YWxgIGlzIGEgcGFyYW1ldGVyIHNwZWNpZmljIHRvIHRoZSBSYW5kb20gRm9yZXN0IG1vZGVsIGFuZCB0aGUgR0JNLiAgCiMgU2V0dGluZyBgc2NvcmVfdHJlZV9pbnRlcnZhbCA9IDVgIHdpbGwgc2NvcmUgdGhlIG1vZGVsIGFmdGVyIGV2ZXJ5IGZpdmUgdHJlZXMuICBUaGUgcGFyYW1ldGVycyB3ZSAKIyBoYXZlIHNldCBiZWxvdyBzcGVjaWZ5IHRoYXQgdGhlIG1vZGVsIHdpbGwgc3RvcCB0cmFpbmluZyBhZnRlciB0aGVyZSBoYXZlIGJlZW4gdGhyZWUgc2NvcmluZyBpbnRlcnZhbHMgCiMgd2hlcmUgdGhlIEFVQyBoYXMgbm90IGluY3JlYXNlZCBtb3JlIHRoYW4gMC4wMDA1LiAgU2luY2Ugd2UgaGF2ZSBzcGVjaWZpZWQgYSB2YWxpZGF0aW9uIGZyYW1lLCAKIyB0aGUgc3RvcHBpbmcgdG9sZXJhbmNlIHdpbGwgYmUgY29tcHV0ZWQgb24gdmFsaWRhdGlvbiBBVUMgcmF0aGVyIHRoYW4gdHJhaW5pbmcgQVVDLiAKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKZ2JtX2ZpdDMgPC0gaDJvLmdibSh4ID0geCwKICAgICAgICAgICAgICAgICAgICB5ID0geSwKICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gImdibV9maXQzIiwKICAgICAgICAgICAgICAgICAgICB2YWxpZGF0aW9uX2ZyYW1lID0gdmFsaWQsICAjb25seSB1c2VkIGlmIHN0b3BwaW5nX3JvdW5kcyA+IDAKICAgICAgICAgICAgICAgICAgICBudHJlZXMgPSA1MDAsCiAgICAgICAgICAgICAgICAgICAgc2NvcmVfdHJlZV9pbnRlcnZhbCA9IDUsICAgICAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfcm91bmRzID0gMywgICAgICAgICAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfbWV0cmljID0gIkFVQyIsICAgICAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfdG9sZXJhbmNlID0gMC4wMDA1LCAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIExldCdzIGNvbXBhcmUgdGhlIHBlcmZvcm1hbmNlIG9mIHRoZSB0d28gR0JNcwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmdibV9wZXJmMSA8LSBoMm8ucGVyZm9ybWFuY2UobW9kZWwgPSBnYm1fZml0MSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXdkYXRhID0gdGVzdCkKZ2JtX3BlcmYyIDwtIGgyby5wZXJmb3JtYW5jZShtb2RlbCA9IGdibV9maXQyLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSB0ZXN0KQpnYm1fcGVyZjMgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gZ2JtX2ZpdDMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbmV3ZGF0YSA9IHRlc3QpCgpgYGAKCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBQcmludCBtb2RlbCBwZXJmb3JtYW5jZQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmdibV9wZXJmMQpnYm1fcGVyZjIKZ2JtX3BlcmYzCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIFJldHJlaXZlIHRlc3Qgc2V0IEFVQwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmgyby5hdWMoZ2JtX3BlcmYxKSAgCmgyby5hdWMoZ2JtX3BlcmYyKSAgCmgyby5hdWMoZ2JtX3BlcmYzKSAgCgpgYGAKCgoKVG8gZXhhbWluZSB0aGUgc2NvcmluZyBoaXN0b3J5LCB1c2UgdGhlIGBzY29yaW5nX2hpc3RvcnlgIG1ldGhvZCBvbiBhIHRyYWluZWQgbW9kZWwuICAKSWYgYHNjb3JlX3RyZWVfaW50ZXJ2YWxgIGlzIG5vdCBzcGVjaWZpZWQsIGl0IHdpbGwgc2NvcmUgYXQgdmFyaW91cyBpbnRlcnZhbHMsIGFzIHdlIGNhbiAKc2VlIGZvciBgaDJvLnNjb3JlSGlzdG9yeSgpYCBiZWxvdy4gIEhvd2V2ZXIsIHJlZ3VsYXIgNS10cmVlIGludGVydmFscyBhcmUgdXNlZCAKZm9yIGBoMm8uc2NvcmVIaXN0b3J5KClgLiAgClRoZSBgZ2JtX2ZpdDJgIHdhcyB0cmFpbmVkIG9ubHkgdXNpbmcgYSB0cmFpbmluZyBzZXQgKG5vIHZhbGlkYXRpb24gc2V0KSwgc28gdGhlIHNjb3JpbmcgCmhpc3RvcnkgaXMgY2FsY3VsYXRlZCBmb3IgdHJhaW5pbmcgc2V0IHBlcmZvcm1hbmNlIG1ldHJpY3Mgb25seS4KCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKaDJvLnNjb3JlSGlzdG9yeShnYm1fZml0MikKCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQojIFdoZW4gZWFybHkgc3RvcHBpbmcgaXMgdXNlZCwgd2Ugc2VlIHRoYXQgdHJhaW5pbmcgc3RvcHBlZCBhdCAxMDUgdHJlZXMgaW5zdGVhZCBvZiB0aGUgZnVsbCA1MDAuICAKIyBTaW5jZSB3ZSB1c2VkIGEgdmFsaWRhdGlvbiBzZXQgaW4gYGdibV9maXQzYCwgYm90aCB0cmFpbmluZyBhbmQgdmFsaWRhdGlvbiBwZXJmb3JtYW5jZSBtZXRyaWNzIAojIGFyZSBzdG9yZWQgaW4gdGhlIHNjb3JpbmcgaGlzdG9yeSBvYmplY3QuICBUYWtlIGEgbG9vayBhdCB0aGUgdmFsaWRhdGlvbiBBVUMgdG8gb2JzZXJ2ZSB0aGF0IHRoZSAKIyBjb3JyZWN0IHN0b3BwaW5nIHRvbGVyYW5jZSB3YXMgZW5mb3JjZWQuCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCgpoMm8uc2NvcmVIaXN0b3J5KGdibV9maXQzKQoKCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKIyBMb29rIGF0IHNjb3JpbmcgaGlzdG9yeSBmb3IgdGhpcmQgR0JNIG1vZGVsCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgpwbG90KGdibV9maXQzLCAKICAgICB0aW1lc3RlcCA9ICJudW1iZXJfb2ZfdHJlZXMiLCAKICAgICBtZXRyaWMgPSAiQVVDIikKcGxvdChnYm1fZml0MywgCiAgICAgdGltZXN0ZXAgPSAibnVtYmVyX29mX3RyZWVzIiwgCiAgICAgbWV0cmljID0gImxvZ2xvc3MiKQoKCgpgYGAKClRoZXJlIGlzIG92ZXJmaXR0aW5nIGZvciBudW1iZXIgb2YgdHJlZXMgZ3JlYXRlciB0aGFuIDUuVGhpcyBjYW4gYmUgb2JzZXJ2ZWQgZnJvbSB0aGUgZ3JhcGggYWJvdmUgd2hpY2ggc2hvd3MgdGhlIHRyYWluaW5nIGVycm9yIGNvbnRpbnVlcyB0byBkZWNyZWFzZSBidXQgdGhlIHZhbGlkYXRpb24gZXJyb3Igc3RhcnRzIHRvIGluY3JlYXNlIGFmdGVyIDUgdHJlZXMuV2Ugd2lsbCBhcnJyaXZlIGF0IGEgYmV0dGVyIG1vZGVsIGJ5IGNob29zaW5nIG51bWJlciBvZiB0cmVlcyB0byBiZSBsZXNzIHRoYW4gNS4KCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKIyA0LiBEZWVwIExlYXJuaW5nCiMgSDJPJ3MgRGVlcCBMZWFybmluZyBhbGdvcml0aG0gaXMgYSBtdWx0aWxheWVyIGZlZWQtZm9yd2FyZCBhcnRpZmljaWFsIG5ldXJhbCBuZXR3b3JrLiAgCiMgSXQgY2FuIGFsc28gYmUgdXNlZCB0byB0cmFpbiBhbiBhdXRvZW5jb2Rlci4gSW4gdGhpcyBleGFtcGxlIHdlIHdpbGwgdHJhaW4gCiMgYSBzdGFuZGFyZCBzdXBlcnZpc2VkIHByZWRpY3Rpb24gbW9kZWwuCgojIFRyYWluIGEgZGVmYXVsdCBETAojIEZpcnN0IHdlIHdpbGwgdHJhaW4gYSBiYXNpYyBETCBtb2RlbCB3aXRoIGRlZmF1bHQgcGFyYW1ldGVycy4gVGhlIERMIG1vZGVsIHdpbGwgaW5mZXIgdGhlIHJlc3BvbnNlIAojIGRpc3RyaWJ1dGlvbiBmcm9tIHRoZSByZXNwb25zZSBlbmNvZGluZyBpZiBpdCBpcyBub3Qgc3BlY2lmaWVkIGV4cGxpY2l0bHkgdGhyb3VnaCB0aGUgYGRpc3RyaWJ1dGlvbmAgCiMgYXJndW1lbnQuICBIMk8ncyBETCB3aWxsIG5vdCBiZSByZXByb2R1Y2libGUgaWYgaXQgaXMgcnVuIG9uIG1vcmUgdGhhbiBhIHNpbmdsZSBjb3JlLCBzbyBpbiB0aGlzIGV4YW1wbGUsIAojIHRoZSBwZXJmb3JtYW5jZSBtZXRyaWNzIGJlbG93IG1heSB2YXJ5IHNsaWdodGx5IGZyb20gd2hhdCB5b3Ugc2VlIG9uIHlvdXIgbWFjaGluZS4KIyBJbiBIMk8ncyBETCwgZWFybHkgc3RvcHBpbmcgaXMgZW5hYmxlZCBieSBkZWZhdWx0LCBzbyBiZWxvdywgaXQgd2lsbCB1c2UgdGhlIHRyYWluaW5nIHNldCBhbmQgCiMgZGVmYXVsdCBzdG9wcGluZyBwYXJhbWV0ZXJzIHRvIHBlcmZvcm0gZWFybHkgc3RvcHBpbmcuCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKZGxfZml0MSA8LSBoMm8uZGVlcGxlYXJuaW5nKHggPSB4LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgeSA9IHksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWxfaWQgPSAiZGxfZml0MSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZWVkID0gMSkKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIFRyYWluIGEgREwgd2l0aCBuZXcgYXJjaGl0ZWN0dXJlIGFuZCBtb3JlIGVwb2Nocy4KIyBOZXh0IHdlIHdpbGwgaW5jcmVhc2UgdGhlIG51bWJlciBvZiBlcG9jaHMgdXNlZCBpbiB0aGUgR0JNIGJ5IHNldHRpbmcgYGVwb2Nocz0yMGAgKHRoZSBkZWZhdWx0IGlzIDEwKS4gIAojIEluY3JlYXNpbmcgdGhlIG51bWJlciBvZiBlcG9jaHMgaW4gYSBkZWVwIG5ldXJhbCBuZXQgbWF5IGluY3JlYXNlIHBlcmZvcm1hbmNlIG9mIHRoZSBtb2RlbCwgaG93ZXZlciwgCiMgeW91IGhhdmUgdG8gYmUgY2FyZWZ1bCBub3QgdG8gb3ZlcmZpdCB5b3VyIG1vZGVsIHRvIHlvdXIgdHJhaW5pbmcgZGF0YS4gIFRvIGF1dG9tYXRpY2FsbHkgZmluZCB0aGUgb3B0aW1hbCBudW1iZXIgb2YgZXBvY2hzLCAKIyB5b3UgbXVzdCB1c2UgSDJPJ3MgZWFybHkgc3RvcHBpbmcgZnVuY3Rpb25hbGl0eS4gIFVubGlrZSB0aGUgcmVzdCBvZiB0aGUgSDJPIGFsZ29yaXRobXMsIEgyTydzIERMIHdpbGwgCiMgdXNlIGVhcmx5IHN0b3BwaW5nIGJ5IGRlZmF1bHQsIHNvIGZvciBjb21wYXJpc29uIHdlIHdpbGwgZmlyc3QgdHVybiBvZmYgZWFybHkgc3RvcHBpbmcuICBXZSBkbyB0aGlzIGluIHRoZSBuZXh0IGV4YW1wbGUgCiMgYnkgc2V0dGluZyBgc3RvcHBpbmdfcm91bmRzPTBgLgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmRsX2ZpdDIgPC0gaDJvLmRlZXBsZWFybmluZyh4ID0geCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHkgPSB5LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gImRsX2ZpdDIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgI3ZhbGlkYXRpb25fZnJhbWUgPSB2YWxpZCwgICNvbmx5IHVzZWQgaWYgc3RvcHBpbmdfcm91bmRzID4gMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZXBvY2hzID0gMjAsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBoaWRkZW49IGMoMTAsMTApLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfcm91bmRzID0gMCwgICMgZGlzYWJsZSBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpCgoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgVHJhaW4gYSBETCB3aXRoIGVhcmx5IHN0b3BwaW5nCiMgVGhpcyBleGFtcGxlIHdpbGwgdXNlIHRoZSBzYW1lIG1vZGVsIHBhcmFtZXRlcnMgYXMgYGRsX2ZpdDJgLiBUaGlzIHRpbWUsIHdlIHdpbGwgdHVybiBvbiAKIyBlYXJseSBzdG9wcGluZyBhbmQgc3BlY2lmeSB0aGUgc3RvcHBpbmcgY3JpdGVyaW9uLiAgV2Ugd2lsbCBhbHNvIHBhc3MgYSB2YWxpZGF0aW9uIHNldCwgYXMgaXMKIyByZWNvbW1lbmRlZCBmb3IgZWFybHkgc3RvcHBpbmcuCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKZGxfZml0MyA8LSBoMm8uZGVlcGxlYXJuaW5nKHggPSB4LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgeSA9IHksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWxfaWQgPSAiZGxfZml0MyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB2YWxpZGF0aW9uX2ZyYW1lID0gdmFsaWQsICAjaW4gREwsIGVhcmx5IHN0b3BwaW5nIGlzIG9uIGJ5IGRlZmF1bHQKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVwb2NocyA9IDIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBoaWRkZW4gPSBjKDEwLDEwKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNjb3JlX2ludGVydmFsID0gMSwgICAgICAgICAgICN1c2VkIGZvciBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfcm91bmRzID0gMywgICAgICAgICAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdG9wcGluZ19tZXRyaWMgPSAiQVVDIiwgICAgICAjdXNlZCBmb3IgZWFybHkgc3RvcHBpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0b3BwaW5nX3RvbGVyYW5jZSA9IDAuMDAwNSwgICN1c2VkIGZvciBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2VlZCA9IDEpCgoKYGBgCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgTGV0J3MgY29tcGFyZSB0aGUgcGVyZm9ybWFuY2Ugb2YgdGhlIHRocmVlIERMIG1vZGVscwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmRsX3BlcmYxIDwtIGgyby5wZXJmb3JtYW5jZShtb2RlbCA9IGRsX2ZpdDEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXdkYXRhID0gdGVzdCkKZGxfcGVyZjIgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gZGxfZml0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSB0ZXN0KQpkbF9wZXJmMyA8LSBoMm8ucGVyZm9ybWFuY2UobW9kZWwgPSBkbF9maXQzLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbmV3ZGF0YSA9IHRlc3QpCgoKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgUHJpbnQgbW9kZWwgcGVyZm9ybWFuY2UKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpkbF9wZXJmMQpkbF9wZXJmMgpkbF9wZXJmMwoKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIFJldHJlaXZlIHRlc3Qgc2V0IEFVQwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCmgyby5hdWMoZGxfcGVyZjEpICAKaDJvLmF1YyhkbF9wZXJmMikgIApoMm8uYXVjKGRsX3BlcmYzKSAgCgoKYGBgCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgU2NvcmluZyBoaXN0b3J5CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKCmgyby5zY29yZUhpc3RvcnkoZGxfZml0MykKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIGNvbmZ1c2lvbiBtYXRyaXgKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpoMm8uY29uZnVzaW9uTWF0cml4KGRsX2ZpdDMpCgpgYGAKCgoKYGBge3J9CiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIG1vZGVsIGRpYWdub3N0aWNzCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgpwbG90KGRsX2ZpdDMsCiAgICAgdGltZXN0ZXAgPSAiZXBvY2hzIiwKICAgICBtZXRyaWMgPSAiY2xhc3NpZmljYXRpb25fZXJyb3IiKQoKCmgyby5zY29yZUhpc3RvcnkoZGxfZml0MykkZXBvY2hzCmgyby5zY29yZUhpc3RvcnkoZGxfZml0MykkdmFsaWRhdGlvbl9jbGFzc2lmaWNhdGlvbl9lcnJvcgpgYGAKCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKIyBMb29rIGF0IHNjb3JpbmcgaGlzdG9yeSBmb3IgdGhpcmQgREwgbW9kZWwKIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KIyBUaGUgbW9kZWwgc3RhcnRzIHRvIG92ZXJmaXR0IGFzIGVwb2NoIGdvZXMgYmV5b25kIDIuIFRoZSB0cmFpbmluZyBlcnJvciBjb250aW51ZXMgdG8gZGVjcmVhc2Ugd2hlcmVhcyB0aGUgCiMgdGVzdCBlcnJvciBiZWdpbnMgdG8gaW5jcmVhc2UuCgpwbG90KGRsX2ZpdDMsIAogICAgIHRpbWVzdGVwID0gImVwb2NocyIsIAogICAgIG1ldHJpYyA9ICJBVUMiKQpgYGAKCgoKCmBgYHtyfQojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKIyAjIEdldCB0aGUgQ1YgbW9kZWxzIGZyb20gdGhlIGBkbF9maXQzYCBvYmplY3QgZm9yIHRoaXJkIERMIG1vZGVsCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKCmRsX2ZpdDMgPC0gaDJvLmRlZXBsZWFybmluZyh4ID0geCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHkgPSB5LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmdfZnJhbWUgPSB0cmFpbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gImRsX2ZpdDMiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsaWRhdGlvbl9mcmFtZSA9IHZhbGlkLCAgI2luIERMLCBlYXJseSBzdG9wcGluZyBpcyBvbiBieSBkZWZhdWx0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICBlcG9jaHMgPSAyLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbmZvbGRzID0gMywKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0b3BwaW5nX21ldHJpYyA9ICJtaXNjbGFzc2lmaWNhdGlvbiIsICN1c2VkIGZvciBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgaGlkZGVuID0gYygxMCwxMCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzY29yZV9pbnRlcnZhbCA9IDEsICAgICAgICAgICAjdXNlZCBmb3IgZWFybHkgc3RvcHBpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHN0b3BwaW5nX3JvdW5kcyA9IDUsICAgICAgICAgICN1c2VkIGZvciBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgI3N0b3BwaW5nX21ldHJpYyA9ICJBVUMiLCAgICAgICN1c2VkIGZvciBlYXJseSBzdG9wcGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RvcHBpbmdfdG9sZXJhbmNlID0gMC4wMDA1LCAgI3VzZWQgZm9yIGVhcmx5IHN0b3BwaW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZWVkID0gMSkKCgpjdl9tb2RlbHMgPC0gc2FwcGx5KGRsX2ZpdDNAbW9kZWwkY3Jvc3NfdmFsaWRhdGlvbl9tb2RlbHMsCiAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24oaSkgaDJvLmdldE1vZGVsKGkkbmFtZSkpCgojIFBsb3QgdGhlIHNjb3JpbmcgaGlzdG9yeSBvdmVyIHRpbWUKcGxvdChjdl9tb2RlbHNbWzJdXSwKICAgICB0aW1lc3RlcCA9ICJlcG9jaHMiLAogICAgIG1ldHJpYyA9ICJjbGFzc2lmaWNhdGlvbl9lcnJvciIpCgpwbG90KGRsX2ZpdDMsCiAgICAgdGltZXN0ZXAgPSAiZXBvY2hzIiwKICAgICBtZXRyaWMgPSAiY2xhc3NpZmljYXRpb25fZXJyb3IiKQoKCmN2X21vZGVsc1tbMV1dCmBgYAoKIEFzIGFuIGFsdGVybmF0aXZlIHRvIG1hbnVhbCB0dW5pbmcsIG9yIOKAnGhhbmQgdHVuaW5n4oCdLCB3ZSBjYW4gdXNlIHRoZSBoMm8uZ3JpZCgpIGZ1bmN0aW9uIHRvIHBlcmZvcm0gZWl0aGVyIGEgQ2FydGVzaWFuIG9yIFJhbmRvbiBHcmlkIFNlYXJjaCAoUkdTKS4gUmFuZG9tIEdyaWQgU2VhcmNoIGlzIHVzdWFsbHkgYSBxdWlja2VyIHdheSB0byBmaW5kIGEgZ29vZCBtb2RlbCwgc28gd2Ugd2lsbCBwcm92aWRlIGEgZXhhbXBsZSBvZiBob3cgdG8gdXNlIEgyT+KAmXMgUmFuZG9tIEdyaWQgU2VhcmNoIG9uIGEgRE5OLgogT25lIGhhbmR5IGZlYXR1cmUgb2YgUkdTIGlzIHRoYXQgeW91IGNhbiBzcGVjaWZ5IGhvdyBsb25nIHlvdSB3b3VsZCBsaWtlIHRvIGV4ZWN1dGUgdGhlIGdyaWQgZm9yIOKAkyB0aGlzIGNhbiBiZSBiYXNlZCBvbiBhIHRpbWUsIG51bWJlciBvZiBtb2RlbHMsIG9yIGEgcGVyZm9ybWFuY2UtbWV0cmljLWJhc2VkIHN0b3BwaW5nIGNyaXRlcmlvbi4gSW4gdGhlIGV4YW1wbGUgYmVsb3csd2Ugd2lsbCB0cmFpbiB0aGUgRE5OIGdyaWQgZm9yIDYwMCBzZWNvbmRzICgxMCBtaW51dGVzKS4KRmlyc3QgZGVmaW5lIGEgZ3JpZCBvZiBEZWVwIExlYXJuaW5nIGh5cGVycGFyYW10ZXJzIGFuZCBzcGVjaWZ5IHRoZSBzZWFyY2hfY3JpdGVyaWEgLgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgRGVlcCBMZWFybmluZyBHcmlkIFNlYXJjaCB0aGlyZCBETCBtb2RlbAojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCgoKYWN0aXZhdGlvbl9vcHQgPC0gYygiUmVjdGlmaWVyIiwgIk1heG91dCIsICJUYW5oIikKbDFfb3B0IDwtIGMoMCwgMC4wMDAwMSwgMC4wMDAxLCAwLjAwMSwgMC4wMSkKbDJfb3B0IDwtIGMoMCwgMC4wMDAwMSwgMC4wMDAxLCAwLjAwMSwgMC4wMSkKaHlwZXJfcGFyYW1zIDwtIGxpc3QoYWN0aXZhdGlvbiA9IGFjdGl2YXRpb25fb3B0LCBsMSA9IGwxX29wdCwgbDIgPSBsMl9vcHQpCnNlYXJjaF9jcml0ZXJpYSA8LSBsaXN0KHN0cmF0ZWd5ID0gIlJhbmRvbURpc2NyZXRlIiwgbWF4X3J1bnRpbWVfc2VjcyA9IDYwMCkKCiMgUmF0aGVyIHRoYW4gY29tcGFyaW5nIG1vZGVscyBieSB1c2luZyBjcm9zcy12YWxpZGF0aW9uICh3aGljaCBpcyDigJxiZXR0ZXLigJ0gYnV0IHRha2VzIGxvbmdlciksIHdlIHdpbGwgc2ltcGx5CiMgcGFydGl0aW9uIG91ciB0cmFpbmluZyBzZXQgaW50byB0d28gcGllY2VzIOKAkyBvbmUgZm9yIHRyYWluaW5nIGFuZCBvbmUgZm9yIHZhbGlkaWF0aW9uLgojIFRoaXMgd2lsbCBzcGxpdCB0aGUgdHJhaW4gZnJhbWUgaW50byBhbiA4MCUgYW5kIDIwJSBwYXJ0aXRpb24gb2YgdGhlIHJvd3MuCgpzcGxpdHMgPC0gaDJvLnNwbGl0RnJhbWUodHJhaW4sIHJhdGlvcyA9IDAuOCwgc2VlZCA9IDEpCgojVHJhaW4gdGhlIHJhbmRvbSBncmlkLiBGaXhlZCBub24tZGVmYXVsdCBwYXJhbWV0ZXJzIHN1Y2ggYXMgaGlkZGVuPWMoMjAsMjApIGNhbiBiZSBwYXNzZWQgZGlyZWN0bHkgdG8KI3RoZSBoMm8uZ3JpZCgpIGZ1bmN0aW9uLgoKCmRsX2dyaWQgPC0gaDJvLmdyaWQoImRlZXBsZWFybmluZyIsIHggPSB4LCB5ID0geSwKICAgICAgICAgICAgICAgICAgICBncmlkX2lkID0gImRsX2dyaWQiLAogICAgICAgICAgICAgICAgICAgIHRyYWluaW5nX2ZyYW1lID0gc3BsaXRzW1sxXV0sCiAgICAgICAgICAgICAgICAgICAgdmFsaWRhdGlvbl9mcmFtZSA9IHNwbGl0c1tbMl1dLAogICAgICAgICAgICAgICAgICAgIHNlZWQgPSAxLAogICAgICAgICAgICAgICAgICAgIGhpZGRlbiA9IGMoMjAsMjApLAogICAgICAgICAgICAgICAgICAgIGh5cGVyX3BhcmFtcyA9IGh5cGVyX3BhcmFtcywKICAgICAgICAgICAgICAgICAgICBzZWFyY2hfY3JpdGVyaWEgPSBzZWFyY2hfY3JpdGVyaWEpCgojT25jZSB3ZSBoYXZlIHRyYWluZWQgdGhlIGdyaWQsIHdlIGNhbiBjb2xsZWN0IHRoZSByZXN1bHRzIGFuZCBzb3J0IGJ5IG91ciBtb2RlbCBwZXJmb3JtYW5jZSBtZXRyaWMgb2YgY2hvaWNlLgoKZGxfZ3JpZHBlcmYgPC0gaDJvLmdldEdyaWQoZ3JpZF9pZCA9ICJkbF9ncmlkIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgc29ydF9ieSA9ICJhY2N1cmFjeSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgIGRlY3JlYXNpbmcgPSBUUlVFKQpwcmludChkbF9ncmlkcGVyZikKYGBgCgoKCgpgYGB7cn0KI0dyYWIgdGhlIG1vZGVsX2lkIGZvciB0aGUgdG9wIERMIG1vZGVsLCBjaG9zZW4gYnkgdmFsaWRhdGlvbiBlcnJvci4KCmJlc3RfZGxfbW9kZWxfaWQgPC0gZGxfZ3JpZHBlcmZAbW9kZWxfaWRzW1sxXV0KYmVzdF9kbCA8LSBoMm8uZ2V0TW9kZWwoYmVzdF9kbF9tb2RlbF9pZCkKCiNOb3cgbGV04oCZcyBldmFsdWF0ZSB0aGUgbW9kZWwgcGVyZm9ybWFuY2Ugb24gYSB0ZXN0IHNldCBzbyB3ZSBnZXQgYW4gaG9uZXN0IGVzdGltYXRlIG9mIHRvcCBtb2RlbAojcGVyZm9ybWFuY2UuCgpiZXN0X2RsX3BlcmYgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gYmVzdF9kbCwgbmV3ZGF0YSA9IHRlc3QpCmgyby5tc2UoYmVzdF9kbF9wZXJmKQoKYGBgCgoKCgpgYGB7cn0KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCiMgNS4gTmFpdmUgQmF5ZXMgbW9kZWwKIyBUaGUgTmFpdmUgQmF5ZXMgKE5CKSBhbGdvcml0aG0gZG9lcyBub3QgdXN1YWxseSBiZWF0IGFuIGFsZ29yaXRobSBsaWtlIGEgUmFuZG9tIEZvcmVzdCAKIyBvciBHQk0sIGhvd2V2ZXIgaXQgaXMgc3RpbGwgYSBwb3B1bGFyIGFsZ29yaXRobSwgZXNwZWNpYWxseSBpbiB0aGUgdGV4dCBkb21haW4gKHdoZW4geW91ciAKIyBpbnB1dCBpcyB0ZXh0IGVuY29kZWQgYXMgIkJhZyBvZiBXb3JkcyIsIGZvciBleGFtcGxlKS4gIFRoZSBOYWl2ZSBCYXllcyBhbGdvcml0aG0gaXMgZm9yIAojIGJpbmFyeSBvciBtdWx0aWNsYXNzIGNsYXNzaWZpY2F0aW9uIHByb2JsZW1zIG9ubHksIG5vdCByZWdyZXNzaW9uLiAgVGhlcmVmb3JlLCB5b3VyIHJlc3BvbnNlIAojIG11c3QgYmUgYSBmYWN0b3IgaW5zdGVhZCBvZiBhIG51bWVyaWMuCgojIEZpcnN0IHdlIHdpbGwgdHJhaW4gYSBiYXNpYyBOQiBtb2RlbCB3aXRoIGRlZmF1bHQgcGFyYW1ldGVycy4gCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKbmJfZml0MSA8LSBoMm8ubmFpdmVCYXllcyh4ID0geCwKICAgICAgICAgICAgICAgICAgICAgICAgICB5ID0geSwKICAgICAgICAgICAgICAgICAgICAgICAgICB0cmFpbmluZ19mcmFtZSA9IHRyYWluLAogICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2lkID0gIm5iX2ZpdDEiKQoKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIFRyYWluIGEgTkIgbW9kZWwgd2l0aCBMYXBsYWNlIFNtb290aGluZwojIE9uZSBvZiB0aGUgZmV3IHR1bmFibGUgbW9kZWwgcGFyYW1ldGVycyBmb3IgdGhlIE5haXZlIEJheWVzIGFsZ29yaXRobSBpcyB0aGUgYW1vdW50IG9mIExhcGxhY2UgCiMgc21vb3RoaW5nLiBUaGUgSDJPIE5haXZlIEJheWVzIG1vZGVsIHdpbGwgbm90IHVzZSBhbnkgTGFwbGFjZSBzbW9vdGhpbmcgYnkgZGVmYXVsdC4KIz09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0KCgpuYl9maXQyIDwtIGgyby5uYWl2ZUJheWVzKHggPSB4LAogICAgICAgICAgICAgICAgICAgICAgICAgIHkgPSB5LAogICAgICAgICAgICAgICAgICAgICAgICAgIHRyYWluaW5nX2ZyYW1lID0gdHJhaW4sCiAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWxfaWQgPSAibmJfZml0MiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgbGFwbGFjZSA9IDYpCgojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKIyBMZXQncyBjb21wYXJlIHRoZSBwZXJmb3JtYW5jZSBvZiB0aGUgdHdvIE5CIG1vZGVscwojPT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PQoKCm5iX3BlcmYxIDwtIGgyby5wZXJmb3JtYW5jZShtb2RlbCA9IG5iX2ZpdDEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBuZXdkYXRhID0gdGVzdCkKbmJfcGVyZjIgPC0gaDJvLnBlcmZvcm1hbmNlKG1vZGVsID0gbmJfZml0MiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSB0ZXN0KQoKCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgojIFByaW50IG1vZGVsIHBlcmZvcm1hbmNlCiM9PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09CgoKbmJfcGVyZjEKbmJfcGVyZjIKCgpgYGAKCg==