Introduction

Mục tiêu của post này là:

  1. Khảo sát chất lượng phân loại của các thuật toán Machine Learning (ML) khác nhau.
  2. Đánh giá tác động của AUC lên Sensitivity và Specificity.
  3. Đánh giá mối liên hệ giữa Specificity và Sensitivity.

Để so sánh, 14 cách tiếp cận của ML được sử dụng cùng với Logistic - một cách tiếp cận của thống kê truyền thống. Kết quả của mô hình Logistic (kí hiệu glm) được sử dụng như chuẩn (base line) so sánh và đánh giá 14 cách tiếp cận còn lại. Dữ liệu có thể download ở đây.

Results and Discussion

Kết quả thử nghiệm trên chạy 15 lần mẫu ngẫu nghiên cho thấy:

  1. Random Forest (ranger, rf) có AUC cao nhất nhưng phân biệt chính xác nhất nhãn Bad thì C5.0. Cách tiếp cận truyền thống là Logistic - một cách tiếp cận thường được sử dụng cho kết quả không khả quan so với các cách tiếp cận khác của ML:

  1. AUC có ảnh hưởng mạnh đến khả năng phân biệt nhãn Bad và có tương quan rất chặt (R2 = 81%):

  1. AUC ảnh hưởng rất yếu đến khả năng phân biệt nhãn Good (R2 chỉ 1%):

  1. Một lần nữa chúng ta có thể xác nhận nguyên lí đánh đổi “khả năng phân biệt càng tốt nhãn Bad thì phân biệt càng tệ nhãn Good (và ngược lại)” (hệ số hồi quy là -3.25):

R Codes

R codes cho các kết quả trên được trình bày dưới đây:

#------------------------------------
# Perform some data pre-processing
#------------------------------------
rm(list = ls())

# Load some packages: 
library(tidyverse)
library(magrittr)

# Import data: 
hmeq <- read.csv("http://www.creditriskanalytics.net/uploads/1/9/5/1/19511601/hmeq.csv")

# Function replaces NA by mean: 

replace_by_mean <- function(x) {
  x[is.na(x)] <- mean(x, na.rm = TRUE)
  return(x)
}

# A function imputes NA observations for categorical variables: 

replace_na_categorical <- function(x) {
  x %>% 
    table() %>% 
    as.data.frame() %>% 
    arrange(-Freq) ->> my_df
  
  n_obs <- sum(my_df$Freq)
  pop <- my_df$. %>% as.character()
  set.seed(29)
  x[is.na(x)] <- sample(pop, sum(is.na(x)), replace = TRUE, prob = my_df$Freq)
  return(x)
}

# Use the two functions: 
df <- hmeq %>% 
  mutate_if(is.factor, as.character) %>% 
  mutate(REASON = case_when(REASON == "" ~ NA_character_, TRUE ~ REASON), 
         JOB = case_when(JOB == "" ~ NA_character_, TRUE ~ JOB)) %>%
  mutate_if(is_character, as.factor) %>% 
  mutate_if(is.numeric, replace_by_mean) %>% 
  mutate_if(is.factor, replace_na_categorical)


# Convert BAD to factor and scale 0 -1 data set: 
df_for_ml <- df %>% 
  mutate(BAD = case_when(BAD == 1 ~ "Bad", TRUE ~ "Good") %>% as.factor()) %>% 
  mutate_if(is.numeric, function(x) {(x - min(x)) / (max(x) - min(x))})



#-----------------------------------
#  Simultaneously Train 10 Models
#-----------------------------------

# Split data: 
library(caret)
set.seed(1)
id <- createDataPartition(y = df_for_ml$BAD, p = 0.7, list = FALSE)
df_train_ml <- df_for_ml[id, ]
df_test_ml <- df_for_ml[-id, ]

# Set conditions for training model and cross-validation: 

set.seed(1)
number <- 3
repeats <- 5
control <- trainControl(method = "repeatedcv", 
                        number = number , 
                        repeats = repeats, 
                        classProbs = TRUE, 
                        savePredictions = "final", 
                        index = createResample(df_train_ml$BAD, repeats*number), 
                        summaryFunction = twoClassSummary, 
                        allowParallel = TRUE)

# Use Parallel computing: 
library(doParallel)
registerDoParallel(cores = detectCores() - 1)

# 16 models selected: 

my_models <- c("adaboost", "xgbTree", "svmRadial", 
               "knn",  "gbm", "C5.0", "ranger",
               "rf", "nnet", "glm", "lda", "treebag", 
               "bagFDA", "glmboost", "cforest")

# Train these ML Models: 

library(caretEnsemble)
set.seed(1)
system.time(model_list1 <- caretList(BAD ~., 
                                     data = df_train_ml,
                                     trControl = control,
                                     metric = "ROC", 
                                     methodList = my_models))


# Extract results: 
list_of_results <- lapply(my_models, function(x) {model_list1[[x]]$resample})

# Convert to data frame: 

total_df <- do.call("bind_rows", list_of_results)
total_df %<>% mutate(Model = lapply(my_models, function(x) {rep(x, number*repeats)}) %>% unlist())

total_df %>% 
  dplyr::select(-Resample) %>% 
  group_by(Model) %>% 
  summarise(avg_auc = mean(ROC), avg_sen = mean(Sens), avg_spec = mean(Spec)) %>% 
  ungroup() -> df_results


# Visualize Model Performance: 

library(hrbrthemes)

my_bar <- function(metric_name) {
  
  metric_name <- noquote(metric_name)
  my_colors <- c("#3E606F")
  
  df_results %>% 
    dplyr::select(Model, metric_name) -> df
  
  names(df) <- c("Model", "value")
  
  df %>% 
    arrange(value) %>%
    mutate(Model = factor(Model, levels = Model)) %>%
    mutate(label = paste0(round(100*value, 1), "%")) -> m
  
  m %>% 
    ggplot(aes(Model, value)) +
    geom_col(fill = my_colors, color = my_colors) +
    coord_flip() +
    geom_text(data = m, aes(label = label), hjust = 1.1, color = "white", size = 4) + 
    theme_ft_rc() + 
    theme(panel.grid = element_blank()) + 
    theme(axis.text.x = element_blank()) + 
    theme(axis.text.y = element_text(color = "white", size = 12)) + 
    scale_y_discrete(expand = c(0.01, 0)) + 
    labs(x = NULL, y = NULL)
}


gridExtra::grid.arrange(my_bar("avg_auc") + labs(title = "AUC/ROC"), 
                        my_bar("avg_sen") + labs(title = "Sensitivity"), 
                        my_bar("avg_spec") + labs(title = "Specificity"), 
                        nrow = 1, padding = unit(99, "line"))




r1 <- lm(avg_sen ~ avg_auc, data = df_results) %>% summary()
r2 <- lm(avg_spec ~ avg_auc, data = df_results) %>% summary()
r3 <- lm(avg_sen ~ avg_spec, data = df_results) %>% summary()

extract_r <- function(object_ols) {
  r <- object_ols$r.squared %>% round(2)
  beta <- object_ols$coefficients %>% as.vector() %>% .[2] %>% round(2)
  return(list(r = r, beta = beta))
  
}

extract_r(r1)
extract_r(r2)
extract_r(r3)


library(ggrepel)

# Figure 1: 

df_results %>% 
  ggplot(aes(avg_auc, avg_sen)) + 
  geom_point(color = "firebrick", size = 4) + 
  geom_text_repel(aes(label = Model), color = "white", size = 4.5, force = 10) + 
  geom_smooth(method = "lm", color = "orange", se = FALSE) + 
  theme_ft_rc(axis_title_size = 12) + 
  theme(panel.grid.minor = element_blank()) + 
  scale_x_continuous(labels = scales::percent) + 
  scale_y_continuous(labels = scales::percent) + 
  annotate("text", x = 0.82, y = 0.66, label = "R^2 == 0.81", color = "white", parse = T, size = 5) + 
  annotate("text", x = 0.82, y = 0.66 - 0.03, label = "beta == 2.18", color = "white", parse = T, size = 5) + 
  labs(x = "AUC/ROC", y = "Sensitivity", 
       title = "Figure 1: The Relationship between AUC/ROC and Sensitivity", 
       caption = "Data Source: http://www.creditriskanalytics.net/")


# Figure 2: 
df_results %>% 
  ggplot(aes(avg_auc, avg_spec)) + 
  geom_point(color = "firebrick", size = 4) + 
  geom_text_repel(aes(label = Model), color = "white", size = 4.5, force = 10) + 
  geom_smooth(method = "lm", color = "orange", se = FALSE) + 
  theme_ft_rc(axis_title_size = 12) + 
  theme(panel.grid.minor = element_blank()) + 
  scale_x_continuous(labels = scales::percent) + 
  scale_y_continuous(labels = scales::percent) + 
  annotate("text", x = 0.82, y = 0.99, label = "R^2 == 0.01", color = "white", parse = T, size = 5) + 
  annotate("text", x = 0.82, y = 0.99 - 0.004, label = "beta == 0.02", color = "white", parse = T, size = 5) + 
  labs(x = "AUC/ROC", y = "Specificity", 
       title = "Figure 2: The Relationship between AUC/ROC and Specificity", 
       caption = "Data Source: http://www.creditriskanalytics.net/")


# Figure 3: 
df_results %>% 
  ggplot(aes(avg_sen, avg_spec)) + 
  geom_point(color = "firebrick", size = 4) + 
  geom_text_repel(aes(label = Model), color = "white", size = 4.5, force = 10) + 
  geom_smooth(method = "lm", color = "orange", se = FALSE) + 
  theme_ft_rc(axis_title_size = 12) + 
  theme(panel.grid.minor = element_blank()) + 
  scale_x_continuous(labels = scales::percent) + 
  scale_y_continuous(labels = scales::percent) + 
  annotate("text", x = 0.35, y = 0.99, label = "R^2 == 0.09", color = "white", parse = T, size = 5) + 
  annotate("text", x = 0.35, y = 0.99 - 0.004, label = "beta == -3.25", color = "white", parse = T, size = 5) + 
  labs(x = "Sensitivity", y = "Specificity", 
       title = "Figure 3: The Relationship between Sensitivity and Specificity", 
       caption = "Data Source: http://www.creditriskanalytics.net/")
LS0tDQp0aXRsZTogIkVmZmVjdCBvZiBBVUMgb24gU2Vuc2l0aXZpdHkgYW5kIFNwZWNpZmljaXR5IGJ5IE1hY2hpbmUgTGVhcm5pbmcgTW9kZWwiDQphdXRob3I6ICJOZ3V5ZW4gQ2hpIER1bmciDQpzdWJ0aXRsZTogIlIgZm9yIFBsZWFzdXJlIg0Kb3V0cHV0Og0KICBodG1sX2RvY3VtZW50Og0KICAgIGNvZGVfZG93bmxvYWQ6IHllcw0KICAgIGNvZGVfZm9sZGluZzogaGlkZQ0KICAgIGhpZ2hsaWdodDogemVuYnVybg0KICAgIHRoZW1lOiBmbGF0bHkNCiAgICB0b2M6IHllcw0KICAgIHRvY19mbG9hdDogeWVzDQogIHdvcmRfZG9jdW1lbnQ6DQogICAgdG9jOiB5ZXMNCi0tLQ0KDQpgYGB7ciBzZXR1cCxpbmNsdWRlPUZBTFNFfQ0Ka25pdHI6Om9wdHNfY2h1bmskc2V0KGVjaG8gPSBUUlVFLCB3YXJuaW5nID0gRkFMU0UsIG1lc3NhZ2UgPSBGQUxTRSwgZmlnLnJldGluYT0yKQ0KYGBgDQoNCg0KIyBJbnRyb2R1Y3Rpb24NCg0KTeG7pWMgdGnDqnUgY+G7p2EgcG9zdCBuw6B5IGzDoDogDQoNCjEuIEto4bqjbyBzw6F0IGNo4bqldCBsxrDhu6NuZyBwaMOibiBsb+G6oWkgY+G7p2EgY8OhYyB0aHXhuq10IHRvw6FuIE1hY2hpbmUgTGVhcm5pbmcgKE1MKSBraMOhYyBuaGF1LiANCjIuIMSQw6FuaCBnacOhIHTDoWMgxJHhu5luZyBj4bunYSBBVUMgbMOqbiBTZW5zaXRpdml0eSB2w6AgU3BlY2lmaWNpdHkuIA0KMy4gxJDDoW5oIGdpw6EgbeG7kWkgbGnDqm4gaOG7hyBnaeG7r2EgU3BlY2lmaWNpdHkgdsOgIFNlbnNpdGl2aXR5LiANCg0KxJDhu4Mgc28gc8OhbmgsIDE0IGPDoWNoIHRp4bq/cCBj4bqtbiBj4bunYSBNTCDEkcaw4bujYyBz4butIGThu6VuZyBjw7luZyB24bubaSBMb2dpc3RpYyAtIG3hu5l0IGPDoWNoIHRp4bq/cCBj4bqtbiBj4bunYSB0aOG7kW5nIGvDqiB0cnV54buBbiB0aOG7kW5nLiBL4bq/dCBxdeG6oyBj4bunYSBtw7QgaMOsbmggTG9naXN0aWMgKGvDrSBoaeG7h3UgZ2xtKSDEkcaw4bujYyBz4butIGThu6VuZyBuaMawIGNodeG6qW4gKGJhc2UgbGluZSkgc28gc8OhbmggdsOgIMSRw6FuaCBnacOhIDE0IGPDoWNoIHRp4bq/cCBj4bqtbiBjw7JuIGzhuqFpLiBE4buvIGxp4buHdSBjw7MgdGjhu4MgZG93bmxvYWQgW+G7nyDEkcOieV0oaHR0cDovL3d3dy5jcmVkaXRyaXNrYW5hbHl0aWNzLm5ldC91cGxvYWRzLzEvOS81LzEvMTk1MTE2MDEvaG1lcS5jc3YpLiANCg0KIyBSZXN1bHRzIGFuZCBEaXNjdXNzaW9uDQoNCkvhur90IHF14bqjIHRo4butIG5naGnhu4dtIHRyw6puIGNo4bqheSAxNSBs4bqnbiBt4bqrdSBuZ+G6q3UgbmdoacOqbiBjaG8gdGjhuqV5OiANCg0KMS4gUmFuZG9tIEZvcmVzdCAocmFuZ2VyLCByZikgY8OzIEFVQyBjYW8gbmjhuqV0IG5oxrBuZyBwaMOibiBiaeG7h3QgY2jDrW5oIHjDoWMgbmjhuqV0IG5ow6NuIEJhZCB0aMOsIEM1LjAuIEPDoWNoIHRp4bq/cCBj4bqtbiB0cnV54buBbiB0aOG7kW5nIGzDoCBMb2dpc3RpYyAtIG3hu5l0IGPDoWNoIHRp4bq/cCBj4bqtbiB0aMaw4budbmcgxJHGsOG7o2Mgc+G7rSBk4bulbmcgY2hvIGvhur90IHF14bqjIGtow7RuZyBraOG6oyBxdWFuIHNvIHbhu5tpIGPDoWMgY8OhY2ggdGnhur9wIGPhuq1uIGtow6FjIGPhu6dhIE1MOiANCg0KDQohW10oQzpcVXNlcnNcWmJvb2tcRGVza3RvcFxwaWNccDEuanBnKQ0KDQoyLiBBVUMgY8OzIOG6o25oIGjGsOG7n25nIG3huqFuaCDEkeG6v24ga2jhuqMgbsSDbmcgcGjDom4gYmnhu4d0IG5ow6NuIEJhZCB2w6AgY8OzIHTGsMahbmcgcXVhbiBy4bqldCBjaOG6t3QgKFIyID0gODElKTogDQoNCiFbXShDOlxVc2Vyc1xaYm9va1xEZXNrdG9wXHBpY1xwMi5qcGcpDQoNCjMuIEFVQyDhuqNuaCBoxrDhu59uZyBy4bqldCB54bq/dSDEkeG6v24ga2jhuqMgbsSDbmcgcGjDom4gYmnhu4d0IG5ow6NuIEdvb2QgKFIyIGNo4buJIDElKTogDQoNCiFbXShDOlxVc2Vyc1xaYm9va1xEZXNrdG9wXHBpY1xwMy5qcGcpDQoNCjQuIE3hu5l0IGzhuqduIG7hu69hIGNow7puZyB0YSBjw7MgdGjhu4MgeMOhYyBuaOG6rW4gbmd1ecOqbiBsw60gxJHDoW5oIMSR4buVaSAqKiJraOG6oyBuxINuZyBwaMOibiBiaeG7h3QgY8OgbmcgdOG7kXQgbmjDo24gQmFkIHRow6wgcGjDom4gYmnhu4d0IGPDoG5nIHThu4cgbmjDo24gR29vZCAodsOgIG5nxrDhu6NjIGzhuqFpKSIqKiAoaOG7hyBz4buRIGjhu5NpIHF1eSBsw6AgLTMuMjUpOiANCg0KDQohW10oQzpcVXNlcnNcWmJvb2tcRGVza3RvcFxwaWNccDQuanBnKQ0KDQojIFIgQ29kZXMNCg0KUiBjb2RlcyBjaG8gY8OhYyBr4bq/dCBxdeG6oyB0csOqbiDEkcaw4bujYyB0csOsbmggYsOgeSBkxrDhu5tpIMSRw6J5OiANCg0KYGBge3IsIGV2YWw9RkFMU0V9DQoNCg0KIy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQ0KIyBQZXJmb3JtIHNvbWUgZGF0YSBwcmUtcHJvY2Vzc2luZw0KIy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQ0Kcm0obGlzdCA9IGxzKCkpDQoNCiMgTG9hZCBzb21lIHBhY2thZ2VzOiANCmxpYnJhcnkodGlkeXZlcnNlKQ0KbGlicmFyeShtYWdyaXR0cikNCg0KIyBJbXBvcnQgZGF0YTogDQpobWVxIDwtIHJlYWQuY3N2KCJodHRwOi8vd3d3LmNyZWRpdHJpc2thbmFseXRpY3MubmV0L3VwbG9hZHMvMS85LzUvMS8xOTUxMTYwMS9obWVxLmNzdiIpDQoNCiMgRnVuY3Rpb24gcmVwbGFjZXMgTkEgYnkgbWVhbjogDQoNCnJlcGxhY2VfYnlfbWVhbiA8LSBmdW5jdGlvbih4KSB7DQogIHhbaXMubmEoeCldIDwtIG1lYW4oeCwgbmEucm0gPSBUUlVFKQ0KICByZXR1cm4oeCkNCn0NCg0KIyBBIGZ1bmN0aW9uIGltcHV0ZXMgTkEgb2JzZXJ2YXRpb25zIGZvciBjYXRlZ29yaWNhbCB2YXJpYWJsZXM6IA0KDQpyZXBsYWNlX25hX2NhdGVnb3JpY2FsIDwtIGZ1bmN0aW9uKHgpIHsNCiAgeCAlPiUgDQogICAgdGFibGUoKSAlPiUgDQogICAgYXMuZGF0YS5mcmFtZSgpICU+JSANCiAgICBhcnJhbmdlKC1GcmVxKSAtPj4gbXlfZGYNCiAgDQogIG5fb2JzIDwtIHN1bShteV9kZiRGcmVxKQ0KICBwb3AgPC0gbXlfZGYkLiAlPiUgYXMuY2hhcmFjdGVyKCkNCiAgc2V0LnNlZWQoMjkpDQogIHhbaXMubmEoeCldIDwtIHNhbXBsZShwb3AsIHN1bShpcy5uYSh4KSksIHJlcGxhY2UgPSBUUlVFLCBwcm9iID0gbXlfZGYkRnJlcSkNCiAgcmV0dXJuKHgpDQp9DQoNCiMgVXNlIHRoZSB0d28gZnVuY3Rpb25zOiANCmRmIDwtIGhtZXEgJT4lIA0KICBtdXRhdGVfaWYoaXMuZmFjdG9yLCBhcy5jaGFyYWN0ZXIpICU+JSANCiAgbXV0YXRlKFJFQVNPTiA9IGNhc2Vfd2hlbihSRUFTT04gPT0gIiIgfiBOQV9jaGFyYWN0ZXJfLCBUUlVFIH4gUkVBU09OKSwgDQogICAgICAgICBKT0IgPSBjYXNlX3doZW4oSk9CID09ICIiIH4gTkFfY2hhcmFjdGVyXywgVFJVRSB+IEpPQikpICU+JQ0KICBtdXRhdGVfaWYoaXNfY2hhcmFjdGVyLCBhcy5mYWN0b3IpICU+JSANCiAgbXV0YXRlX2lmKGlzLm51bWVyaWMsIHJlcGxhY2VfYnlfbWVhbikgJT4lIA0KICBtdXRhdGVfaWYoaXMuZmFjdG9yLCByZXBsYWNlX25hX2NhdGVnb3JpY2FsKQ0KDQoNCiMgQ29udmVydCBCQUQgdG8gZmFjdG9yIGFuZCBzY2FsZSAwIC0xIGRhdGEgc2V0OiANCmRmX2Zvcl9tbCA8LSBkZiAlPiUgDQogIG11dGF0ZShCQUQgPSBjYXNlX3doZW4oQkFEID09IDEgfiAiQmFkIiwgVFJVRSB+ICJHb29kIikgJT4lIGFzLmZhY3RvcigpKSAlPiUgDQogIG11dGF0ZV9pZihpcy5udW1lcmljLCBmdW5jdGlvbih4KSB7KHggLSBtaW4oeCkpIC8gKG1heCh4KSAtIG1pbih4KSl9KQ0KDQoNCg0KIy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tDQojICBTaW11bHRhbmVvdXNseSBUcmFpbiAxMCBNb2RlbHMNCiMtLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLQ0KDQojIFNwbGl0IGRhdGE6IA0KbGlicmFyeShjYXJldCkNCnNldC5zZWVkKDEpDQppZCA8LSBjcmVhdGVEYXRhUGFydGl0aW9uKHkgPSBkZl9mb3JfbWwkQkFELCBwID0gMC43LCBsaXN0ID0gRkFMU0UpDQpkZl90cmFpbl9tbCA8LSBkZl9mb3JfbWxbaWQsIF0NCmRmX3Rlc3RfbWwgPC0gZGZfZm9yX21sWy1pZCwgXQ0KDQojIFNldCBjb25kaXRpb25zIGZvciB0cmFpbmluZyBtb2RlbCBhbmQgY3Jvc3MtdmFsaWRhdGlvbjogDQoNCnNldC5zZWVkKDEpDQpudW1iZXIgPC0gMw0KcmVwZWF0cyA8LSA1DQpjb250cm9sIDwtIHRyYWluQ29udHJvbChtZXRob2QgPSAicmVwZWF0ZWRjdiIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgbnVtYmVyID0gbnVtYmVyICwgDQogICAgICAgICAgICAgICAgICAgICAgICByZXBlYXRzID0gcmVwZWF0cywgDQogICAgICAgICAgICAgICAgICAgICAgICBjbGFzc1Byb2JzID0gVFJVRSwgDQogICAgICAgICAgICAgICAgICAgICAgICBzYXZlUHJlZGljdGlvbnMgPSAiZmluYWwiLCANCiAgICAgICAgICAgICAgICAgICAgICAgIGluZGV4ID0gY3JlYXRlUmVzYW1wbGUoZGZfdHJhaW5fbWwkQkFELCByZXBlYXRzKm51bWJlciksIA0KICAgICAgICAgICAgICAgICAgICAgICAgc3VtbWFyeUZ1bmN0aW9uID0gdHdvQ2xhc3NTdW1tYXJ5LCANCiAgICAgICAgICAgICAgICAgICAgICAgIGFsbG93UGFyYWxsZWwgPSBUUlVFKQ0KDQojIFVzZSBQYXJhbGxlbCBjb21wdXRpbmc6IA0KbGlicmFyeShkb1BhcmFsbGVsKQ0KcmVnaXN0ZXJEb1BhcmFsbGVsKGNvcmVzID0gZGV0ZWN0Q29yZXMoKSAtIDEpDQoNCiMgMTYgbW9kZWxzIHNlbGVjdGVkOiANCg0KbXlfbW9kZWxzIDwtIGMoImFkYWJvb3N0IiwgInhnYlRyZWUiLCAic3ZtUmFkaWFsIiwgDQogICAgICAgICAgICAgICAia25uIiwgICJnYm0iLCAiQzUuMCIsICJyYW5nZXIiLA0KICAgICAgICAgICAgICAgInJmIiwgIm5uZXQiLCAiZ2xtIiwgImxkYSIsICJ0cmVlYmFnIiwgDQogICAgICAgICAgICAgICAiYmFnRkRBIiwgImdsbWJvb3N0IiwgImNmb3Jlc3QiKQ0KDQojIFRyYWluIHRoZXNlIE1MIE1vZGVsczogDQoNCmxpYnJhcnkoY2FyZXRFbnNlbWJsZSkNCnNldC5zZWVkKDEpDQpzeXN0ZW0udGltZShtb2RlbF9saXN0MSA8LSBjYXJldExpc3QoQkFEIH4uLCANCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhID0gZGZfdHJhaW5fbWwsDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJDb250cm9sID0gY29udHJvbCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtZXRyaWMgPSAiUk9DIiwgDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWV0aG9kTGlzdCA9IG15X21vZGVscykpDQoNCg0KIyBFeHRyYWN0IHJlc3VsdHM6IA0KbGlzdF9vZl9yZXN1bHRzIDwtIGxhcHBseShteV9tb2RlbHMsIGZ1bmN0aW9uKHgpIHttb2RlbF9saXN0MVtbeF1dJHJlc2FtcGxlfSkNCg0KIyBDb252ZXJ0IHRvIGRhdGEgZnJhbWU6IA0KDQp0b3RhbF9kZiA8LSBkby5jYWxsKCJiaW5kX3Jvd3MiLCBsaXN0X29mX3Jlc3VsdHMpDQp0b3RhbF9kZiAlPD4lIG11dGF0ZShNb2RlbCA9IGxhcHBseShteV9tb2RlbHMsIGZ1bmN0aW9uKHgpIHtyZXAoeCwgbnVtYmVyKnJlcGVhdHMpfSkgJT4lIHVubGlzdCgpKQ0KDQp0b3RhbF9kZiAlPiUgDQogIGRwbHlyOjpzZWxlY3QoLVJlc2FtcGxlKSAlPiUgDQogIGdyb3VwX2J5KE1vZGVsKSAlPiUgDQogIHN1bW1hcmlzZShhdmdfYXVjID0gbWVhbihST0MpLCBhdmdfc2VuID0gbWVhbihTZW5zKSwgYXZnX3NwZWMgPSBtZWFuKFNwZWMpKSAlPiUgDQogIHVuZ3JvdXAoKSAtPiBkZl9yZXN1bHRzDQoNCg0KIyBWaXN1YWxpemUgTW9kZWwgUGVyZm9ybWFuY2U6IA0KDQpsaWJyYXJ5KGhyYnJ0aGVtZXMpDQoNCm15X2JhciA8LSBmdW5jdGlvbihtZXRyaWNfbmFtZSkgew0KICANCiAgbWV0cmljX25hbWUgPC0gbm9xdW90ZShtZXRyaWNfbmFtZSkNCiAgbXlfY29sb3JzIDwtIGMoIiMzRTYwNkYiKQ0KICANCiAgZGZfcmVzdWx0cyAlPiUgDQogICAgZHBseXI6OnNlbGVjdChNb2RlbCwgbWV0cmljX25hbWUpIC0+IGRmDQogIA0KICBuYW1lcyhkZikgPC0gYygiTW9kZWwiLCAidmFsdWUiKQ0KICANCiAgZGYgJT4lIA0KICAgIGFycmFuZ2UodmFsdWUpICU+JQ0KICAgIG11dGF0ZShNb2RlbCA9IGZhY3RvcihNb2RlbCwgbGV2ZWxzID0gTW9kZWwpKSAlPiUNCiAgICBtdXRhdGUobGFiZWwgPSBwYXN0ZTAocm91bmQoMTAwKnZhbHVlLCAxKSwgIiUiKSkgLT4gbQ0KICANCiAgbSAlPiUgDQogICAgZ2dwbG90KGFlcyhNb2RlbCwgdmFsdWUpKSArDQogICAgZ2VvbV9jb2woZmlsbCA9IG15X2NvbG9ycywgY29sb3IgPSBteV9jb2xvcnMpICsNCiAgICBjb29yZF9mbGlwKCkgKw0KICAgIGdlb21fdGV4dChkYXRhID0gbSwgYWVzKGxhYmVsID0gbGFiZWwpLCBoanVzdCA9IDEuMSwgY29sb3IgPSAid2hpdGUiLCBzaXplID0gNCkgKyANCiAgICB0aGVtZV9mdF9yYygpICsgDQogICAgdGhlbWUocGFuZWwuZ3JpZCA9IGVsZW1lbnRfYmxhbmsoKSkgKyANCiAgICB0aGVtZShheGlzLnRleHQueCA9IGVsZW1lbnRfYmxhbmsoKSkgKyANCiAgICB0aGVtZShheGlzLnRleHQueSA9IGVsZW1lbnRfdGV4dChjb2xvciA9ICJ3aGl0ZSIsIHNpemUgPSAxMikpICsgDQogICAgc2NhbGVfeV9kaXNjcmV0ZShleHBhbmQgPSBjKDAuMDEsIDApKSArIA0KICAgIGxhYnMoeCA9IE5VTEwsIHkgPSBOVUxMKQ0KfQ0KDQoNCmdyaWRFeHRyYTo6Z3JpZC5hcnJhbmdlKG15X2JhcigiYXZnX2F1YyIpICsgbGFicyh0aXRsZSA9ICJBVUMvUk9DIiksIA0KICAgICAgICAgICAgICAgICAgICAgICAgbXlfYmFyKCJhdmdfc2VuIikgKyBsYWJzKHRpdGxlID0gIlNlbnNpdGl2aXR5IiksIA0KICAgICAgICAgICAgICAgICAgICAgICAgbXlfYmFyKCJhdmdfc3BlYyIpICsgbGFicyh0aXRsZSA9ICJTcGVjaWZpY2l0eSIpLCANCiAgICAgICAgICAgICAgICAgICAgICAgIG5yb3cgPSAxLCBwYWRkaW5nID0gdW5pdCg5OSwgImxpbmUiKSkNCg0KDQoNCg0KcjEgPC0gbG0oYXZnX3NlbiB+IGF2Z19hdWMsIGRhdGEgPSBkZl9yZXN1bHRzKSAlPiUgc3VtbWFyeSgpDQpyMiA8LSBsbShhdmdfc3BlYyB+IGF2Z19hdWMsIGRhdGEgPSBkZl9yZXN1bHRzKSAlPiUgc3VtbWFyeSgpDQpyMyA8LSBsbShhdmdfc2VuIH4gYXZnX3NwZWMsIGRhdGEgPSBkZl9yZXN1bHRzKSAlPiUgc3VtbWFyeSgpDQoNCmV4dHJhY3RfciA8LSBmdW5jdGlvbihvYmplY3Rfb2xzKSB7DQogIHIgPC0gb2JqZWN0X29scyRyLnNxdWFyZWQgJT4lIHJvdW5kKDIpDQogIGJldGEgPC0gb2JqZWN0X29scyRjb2VmZmljaWVudHMgJT4lIGFzLnZlY3RvcigpICU+JSAuWzJdICU+JSByb3VuZCgyKQ0KICByZXR1cm4obGlzdChyID0gciwgYmV0YSA9IGJldGEpKQ0KICANCn0NCg0KZXh0cmFjdF9yKHIxKQ0KZXh0cmFjdF9yKHIyKQ0KZXh0cmFjdF9yKHIzKQ0KDQoNCmxpYnJhcnkoZ2dyZXBlbCkNCg0KIyBGaWd1cmUgMTogDQoNCmRmX3Jlc3VsdHMgJT4lIA0KICBnZ3Bsb3QoYWVzKGF2Z19hdWMsIGF2Z19zZW4pKSArIA0KICBnZW9tX3BvaW50KGNvbG9yID0gImZpcmVicmljayIsIHNpemUgPSA0KSArIA0KICBnZW9tX3RleHRfcmVwZWwoYWVzKGxhYmVsID0gTW9kZWwpLCBjb2xvciA9ICJ3aGl0ZSIsIHNpemUgPSA0LjUsIGZvcmNlID0gMTApICsgDQogIGdlb21fc21vb3RoKG1ldGhvZCA9ICJsbSIsIGNvbG9yID0gIm9yYW5nZSIsIHNlID0gRkFMU0UpICsgDQogIHRoZW1lX2Z0X3JjKGF4aXNfdGl0bGVfc2l6ZSA9IDEyKSArIA0KICB0aGVtZShwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpKSArIA0KICBzY2FsZV94X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50KSArIA0KICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50KSArIA0KICBhbm5vdGF0ZSgidGV4dCIsIHggPSAwLjgyLCB5ID0gMC42NiwgbGFiZWwgPSAiUl4yID09IDAuODEiLCBjb2xvciA9ICJ3aGl0ZSIsIHBhcnNlID0gVCwgc2l6ZSA9IDUpICsgDQogIGFubm90YXRlKCJ0ZXh0IiwgeCA9IDAuODIsIHkgPSAwLjY2IC0gMC4wMywgbGFiZWwgPSAiYmV0YSA9PSAyLjE4IiwgY29sb3IgPSAid2hpdGUiLCBwYXJzZSA9IFQsIHNpemUgPSA1KSArIA0KICBsYWJzKHggPSAiQVVDL1JPQyIsIHkgPSAiU2Vuc2l0aXZpdHkiLCANCiAgICAgICB0aXRsZSA9ICJGaWd1cmUgMTogVGhlIFJlbGF0aW9uc2hpcCBiZXR3ZWVuIEFVQy9ST0MgYW5kIFNlbnNpdGl2aXR5IiwgDQogICAgICAgY2FwdGlvbiA9ICJEYXRhIFNvdXJjZTogaHR0cDovL3d3dy5jcmVkaXRyaXNrYW5hbHl0aWNzLm5ldC8iKQ0KDQoNCiMgRmlndXJlIDI6IA0KZGZfcmVzdWx0cyAlPiUgDQogIGdncGxvdChhZXMoYXZnX2F1YywgYXZnX3NwZWMpKSArIA0KICBnZW9tX3BvaW50KGNvbG9yID0gImZpcmVicmljayIsIHNpemUgPSA0KSArIA0KICBnZW9tX3RleHRfcmVwZWwoYWVzKGxhYmVsID0gTW9kZWwpLCBjb2xvciA9ICJ3aGl0ZSIsIHNpemUgPSA0LjUsIGZvcmNlID0gMTApICsgDQogIGdlb21fc21vb3RoKG1ldGhvZCA9ICJsbSIsIGNvbG9yID0gIm9yYW5nZSIsIHNlID0gRkFMU0UpICsgDQogIHRoZW1lX2Z0X3JjKGF4aXNfdGl0bGVfc2l6ZSA9IDEyKSArIA0KICB0aGVtZShwYW5lbC5ncmlkLm1pbm9yID0gZWxlbWVudF9ibGFuaygpKSArIA0KICBzY2FsZV94X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50KSArIA0KICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50KSArIA0KICBhbm5vdGF0ZSgidGV4dCIsIHggPSAwLjgyLCB5ID0gMC45OSwgbGFiZWwgPSAiUl4yID09IDAuMDEiLCBjb2xvciA9ICJ3aGl0ZSIsIHBhcnNlID0gVCwgc2l6ZSA9IDUpICsgDQogIGFubm90YXRlKCJ0ZXh0IiwgeCA9IDAuODIsIHkgPSAwLjk5IC0gMC4wMDQsIGxhYmVsID0gImJldGEgPT0gMC4wMiIsIGNvbG9yID0gIndoaXRlIiwgcGFyc2UgPSBULCBzaXplID0gNSkgKyANCiAgbGFicyh4ID0gIkFVQy9ST0MiLCB5ID0gIlNwZWNpZmljaXR5IiwgDQogICAgICAgdGl0bGUgPSAiRmlndXJlIDI6IFRoZSBSZWxhdGlvbnNoaXAgYmV0d2VlbiBBVUMvUk9DIGFuZCBTcGVjaWZpY2l0eSIsIA0KICAgICAgIGNhcHRpb24gPSAiRGF0YSBTb3VyY2U6IGh0dHA6Ly93d3cuY3JlZGl0cmlza2FuYWx5dGljcy5uZXQvIikNCg0KDQojIEZpZ3VyZSAzOiANCmRmX3Jlc3VsdHMgJT4lIA0KICBnZ3Bsb3QoYWVzKGF2Z19zZW4sIGF2Z19zcGVjKSkgKyANCiAgZ2VvbV9wb2ludChjb2xvciA9ICJmaXJlYnJpY2siLCBzaXplID0gNCkgKyANCiAgZ2VvbV90ZXh0X3JlcGVsKGFlcyhsYWJlbCA9IE1vZGVsKSwgY29sb3IgPSAid2hpdGUiLCBzaXplID0gNC41LCBmb3JjZSA9IDEwKSArIA0KICBnZW9tX3Ntb290aChtZXRob2QgPSAibG0iLCBjb2xvciA9ICJvcmFuZ2UiLCBzZSA9IEZBTFNFKSArIA0KICB0aGVtZV9mdF9yYyhheGlzX3RpdGxlX3NpemUgPSAxMikgKyANCiAgdGhlbWUocGFuZWwuZ3JpZC5taW5vciA9IGVsZW1lbnRfYmxhbmsoKSkgKyANCiAgc2NhbGVfeF9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6cGVyY2VudCkgKyANCiAgc2NhbGVfeV9jb250aW51b3VzKGxhYmVscyA9IHNjYWxlczo6cGVyY2VudCkgKyANCiAgYW5ub3RhdGUoInRleHQiLCB4ID0gMC4zNSwgeSA9IDAuOTksIGxhYmVsID0gIlJeMiA9PSAwLjA5IiwgY29sb3IgPSAid2hpdGUiLCBwYXJzZSA9IFQsIHNpemUgPSA1KSArIA0KICBhbm5vdGF0ZSgidGV4dCIsIHggPSAwLjM1LCB5ID0gMC45OSAtIDAuMDA0LCBsYWJlbCA9ICJiZXRhID09IC0zLjI1IiwgY29sb3IgPSAid2hpdGUiLCBwYXJzZSA9IFQsIHNpemUgPSA1KSArIA0KICBsYWJzKHggPSAiU2Vuc2l0aXZpdHkiLCB5ID0gIlNwZWNpZmljaXR5IiwgDQogICAgICAgdGl0bGUgPSAiRmlndXJlIDM6IFRoZSBSZWxhdGlvbnNoaXAgYmV0d2VlbiBTZW5zaXRpdml0eSBhbmQgU3BlY2lmaWNpdHkiLCANCiAgICAgICBjYXB0aW9uID0gIkRhdGEgU291cmNlOiBodHRwOi8vd3d3LmNyZWRpdHJpc2thbmFseXRpY3MubmV0LyIpDQoNCmBgYA0KDQoNCg==