Introduction

Dữ liệu bất đối xứng (Unbalanced Data / Imbalanced Data) là một thực tế phổ biến và trong những tình huống này thì các tiêu chí đánh giá mô hình như Accuracy là không thể sử dụng được để so sánh và đánh giá chất lượng phân loại của mô hình. Trong những tình huống như vậy, sử dụng các kĩ thuật “tái cân bằng” - giữa các nhãn bằng các kĩ thuật chọn mẫu sau có thể được sử dụng:

  1. Upsampling
  2. Downsampling
  3. SMOTE.

Data and Preprocessing Technique Used

Nhằm đánh giá tác động của các kĩ thuật chọn mẫu áp dụng cho tình huống dữ liệu là bất đối xứng, bộ dữ liệu Caravan được sử dụng và tiêu chí được sử dụng để đánh giá chất lượng phân loại của các thuật toán là AUC.

Dữ liệu được phân chia thành tỉ lệ 70 - 30 (70% cho Training Data) trong đó các biến Near-zeros không được sử dụng và các biến tương quan cao trên 0.9 được loại bỏ theo thuật toán trình bày ở Chapter 3 cuốn Applied Predictive Modelling (Springer Publisher, 2013).

Method Comparison

Ứng với một phương pháp chọn mẫu và một thuật toán được được lựa chọn, 15 bộ dữ liệu Validation được sử dụng để đánh giá AUC. Average AUC của 15 lần chọn mẫu sẽ được sử dụng để đánh giá tác động của các phương pháp chọn mẫu ứng với một thuật toán được chọn.

Kết quả chi ra rằng ảnh hưởng của các phương pháp chọn mẫu lên chất lượng phân loại của mô hình (theo tiêu chí AUC) là không nhất quán với mỗi một thuật toán được sử dụng (hình 1):

Kết quả này chỉ ra rằng, ví dụ, với AdaBoost và Support Vector Machine thì Downsapling làm tăng đáng kể AUC nhưng Naive Bayes Classifier thì gần như “trơ” với các phương pháp chọn mẫu.

Kết quả này là có chút khác biệt so với những kết luận của Win-Vector Blog.

R Codes cho kết quả thu được ở hình 1:

# Load packages and Caravan data set: 
rm(list = ls())
library(ISLR)
library(tidyverse)
library(magrittr)
library(caret)
data("Caravan")

# Function for data processing: 

processingData <- function(...) {
  
  Purchase <- Caravan$Purchase
  df_num <- Caravan %>% select(-Purchase)
  near_zeros <- nearZeroVar(df_num)
  df_num_s1 <- df_num %>% select(-near_zeros)
  highCorrs <- findCorrelation(cor(df_num_s1), cutoff = 0.9)
  df_num_s1 %>% 
    select(-highCorrs) %>% 
    mutate(Purchase = Purchase) %>% 
    return()
}


# Use the function: 

df <- processingData()

# Split data: 
set.seed(1)
id <- createDataPartition(y = df$Purchase, p = 0.7, list = FALSE)
df_train_ml <- df[id, ]
df_test_ml <- df[-id, ]

# Set conditions for training model and cross-validation: 

set.seed(1)
number <- 3
repeats <- 5
control <- trainControl(method = "repeatedcv", 
                        number = number , 
                        repeats = repeats, 
                        classProbs = TRUE, 
                        savePredictions = "final", 
                        index = createResample(df_train_ml$Purchase, repeats*number), 
                        summaryFunction = multiClassSummary, 
                        allowParallel = TRUE)



# Use Parallel computing: 
library(doParallel)
registerDoParallel(cores = detectCores() - 1)

# 6 models selected: 

my_models <- c("rf", "adaboost", "knn", "svmRadial", "glm", "nb")

# Train these ML Models: 
library(caretEnsemble)
set.seed(1)
system.time(model_list1 <- caretList(Purchase ~., 
                                     data = df_train_ml,
                                     trControl = control,
                                     metric = "Accuracy", 
                                     methodList = my_models))


# Function for collecting results: 
getResults <- function(list_models) {
  
  list_of_results <- lapply(my_models, function(x) {list_models[[x]]$resample})
  total_df <- do.call("bind_rows", list_of_results)
  total_df %<>% mutate(Model = lapply(my_models, function(x) {rep(x, number*repeats)}) %>% unlist())
  return(total_df)
}


# Function for comparing Average Accuracy based on 15 samples for these models: 

comparingAUC <- function(df_selected) {
  
  df_selected %>% 
    group_by(Model) %>% 
    summarise(avg_acc = mean(AUC)) %>% 
    ungroup() %>% 
    arrange(-avg_acc) %>% 
    mutate_if(is.numeric, function(x) {round(x, 3)}) %>% 
    return()
}



# Resampling techniques: 

re_sampling <- c("up", "down", "smote")

# Function for training model by resampling technique: 

my_trainBySampling <- function(sampling) {
  control$sampling <- sampling
  model_list <- caretList(Purchase ~., 
                          data = df_train_ml,
                          trControl = control,
                          metric = "Accuracy", 
                          methodList = my_models)
  return(model_list)
}

# Train models by resampling technique: 

lapply(re_sampling, my_trainBySampling) -> all_models


# Compare AUC: 


model_list1 %>% 
  getResults() %>% 
  comparingAUC() %>% 
  mutate(Method = "None") %>% 
  bind_rows(all_models[[1]] %>% getResults() %>% comparingAUC() %>% mutate(Method = "Up")) %>% 
  bind_rows(all_models[[2]] %>% getResults() %>% comparingAUC() %>% mutate(Method = "Down")) %>% 
  bind_rows(all_models[[3]] %>% getResults() %>% comparingAUC() %>% mutate(Method = "SMOTE")) -> df_auc_com


df_auc_com %>% 
  ggplot(aes(Model, avg_acc, fill = Method)) + 
  geom_col(position = "dodge") + 
  scale_fill_manual(values = c('#e41a1c','#377eb8','#4daf4a','#984ea3')) + 
  theme_minimal() + 
  scale_y_continuous(labels = scales::percent, limits = c(0, 0.75), breaks = seq(0, 0.75, by = 0.1)) + 
  scale_x_discrete(labels = c("AdaBoost", "GLM", "KNN", "NB", "RF", "SVM")) + 
  theme(panel.grid.major.x = element_blank()) + 
  labs(x = NULL, y = "Average AUC", 
       subtitle = "Data Source: The Insurance Company Data Provided by Sentient Machine Research", 
       title = "Figure 1: Model Performance by Resample Method Used for Some ML Approaches")
LS0tDQp0aXRsZTogIkRvZXMgQmFsYW5jaW5nIENsYXNzZXMgSW1wcm92ZSBDbGFzc2lmaWVyIFBlcmZvcm1hbmNlPyIgDQpzdWJ0aXRsZTogIlIgZm9yIFBsZWFzZXJlIg0KYXV0aG9yOiAiTmd1eWVuIENoaSBEdW5nIg0Kb3V0cHV0Og0KICBodG1sX2RvY3VtZW50OiANCiAgICBjb2RlX2Rvd25sb2FkOiB0cnVlDQogICAgIyBjb2RlX2ZvbGRpbmc6IGhpZGUNCiAgICBoaWdobGlnaHQ6IHB5Z21lbnRzDQogICAgIyBudW1iZXJfc2VjdGlvbnM6IHllcw0KICAgIHRoZW1lOiAiZmxhdGx5Ig0KICAgIHRvYzogVFJVRQ0KICAgIHRvY19mbG9hdDogVFJVRQ0KLS0tDQoNCmBgYHtyIHNldHVwLGluY2x1ZGU9RkFMU0V9DQprbml0cjo6b3B0c19jaHVuayRzZXQoZWNobyA9IFRSVUUsIHdhcm5pbmcgPSBGQUxTRSwgbWVzc2FnZSA9IEZBTFNFKQ0KYGBgDQoNCiMgSW50cm9kdWN0aW9uDQoNCkThu68gbGnhu4d1IGLhuqV0IMSR4buRaSB44bupbmcgKFVuYmFsYW5jZWQgRGF0YSAvIEltYmFsYW5jZWQgRGF0YSkgbMOgIG3hu5l0IHRo4buxYyB04bq/IHBo4buVIGJp4bq/biB2w6AgdHJvbmcgbmjhu69uZyB0w6xuaCBodeG7kW5nIG7DoHkgdGjDrCBjw6FjIHRpw6p1IGNow60gxJHDoW5oIGdpw6EgbcO0IGjDrG5oIG5oxrAgQWNjdXJhY3kgbMOgIGtow7RuZyB0aOG7gyBz4butIGThu6VuZyDEkcaw4bujYyDEkeG7gyBzbyBzw6FuaCB2w6AgxJHDoW5oIGdpw6EgY2jhuqV0IGzGsOG7o25nIHBow6JuIGxv4bqhaSBj4bunYSBtw7QgaMOsbmguIFRyb25nIG5o4buvbmcgdMOsbmggaHXhu5FuZyBuaMawIHbhuq15LCBz4butIGThu6VuZyBjw6FjIGvEqSB0aHXhuq10ICJ0w6FpIGPDom4gYuG6sW5nIiAtIGdp4buvYSBjw6FjIG5ow6NuIGLhurFuZyBjw6FjIGvEqSB0aHXhuq10IGNo4buNbiBt4bqrdSBzYXUgY8OzIHRo4buDIMSRxrDhu6NjIHPhu60gZOG7pW5nOiANCg0KMS4gVXBzYW1wbGluZw0KMi4gRG93bnNhbXBsaW5nDQozLiBTTU9URS4gDQoNCiMgRGF0YSBhbmQgUHJlcHJvY2Vzc2luZyBUZWNobmlxdWUgVXNlZA0KDQpOaOG6sW0gxJHDoW5oIGdpw6EgdMOhYyDEkeG7mW5nIGPhu6dhIGPDoWMga8SpIHRodeG6rXQgY2jhu41uIG3huqt1IMOhcCBk4bulbmcgY2hvIHTDrG5oIGh14buRbmcgZOG7ryBsaeG7h3UgbMOgIGLhuqV0IMSR4buRaSB44bupbmcsIGLhu5kgZOG7ryBsaeG7h3UgQ2FyYXZhbiDEkcaw4bujYyBz4butIGThu6VuZyB2w6AgdGnDqnUgY2jDrSDEkcaw4bujYyBz4butIGThu6VuZyDEkeG7gyDEkcOhbmggZ2nDoSBjaOG6pXQgbMaw4bujbmcgcGjDom4gbG/huqFpIGPhu6dhIGPDoWMgdGh14bqtdCB0b8OhbiBsw6AgQVVDLiANCg0KROG7ryBsaeG7h3UgxJHGsOG7o2MgcGjDom4gY2hpYSB0aMOgbmggdOG7iSBs4buHIDcwIC0gMzAgKDcwJSBjaG8gVHJhaW5pbmcgRGF0YSkgdHJvbmcgxJHDsyBjw6FjIGJp4bq/biBOZWFyLXplcm9zIGtow7RuZyDEkcaw4bujYyBz4butIGThu6VuZyB2w6AgY8OhYyBiaeG6v24gdMawxqFuZyBxdWFuIGNhbyB0csOqbiAwLjkgxJHGsOG7o2MgbG/huqFpIGLhu48gdGhlbyB0aHXhuq10IHRvw6FuIHRyw6xuaCBiw6B5IOG7nyBDaGFwdGVyIDMgY3Xhu5FuIEFwcGxpZWQgUHJlZGljdGl2ZSBNb2RlbGxpbmcgKFNwcmluZ2VyIFB1Ymxpc2hlciwgMjAxMykuIA0KDQojIE1ldGhvZCBDb21wYXJpc29uDQoNCuG7qG5nIHbhu5tpIG3hu5l0IHBoxrDGoW5nIHBow6FwIGNo4buNbiBt4bqrdSB2w6AgbeG7mXQgdGh14bqtdCB0b8OhbiDEkcaw4bujYyDEkcaw4bujYyBs4buxYSBjaOG7jW4sIDE1IGLhu5kgZOG7ryBsaeG7h3UgVmFsaWRhdGlvbiDEkcaw4bujYyBz4butIGThu6VuZyDEkeG7gyDEkcOhbmggZ2nDoSBBVUMuIEF2ZXJhZ2UgQVVDIGPhu6dhIDE1IGzhuqduIGNo4buNbiBt4bqrdSBz4bq9IMSRxrDhu6NjIHPhu60gZOG7pW5nIMSR4buDIMSRw6FuaCBnacOhIHTDoWMgxJHhu5luZyBj4bunYSBjw6FjIHBoxrDGoW5nIHBow6FwIGNo4buNbiBt4bqrdSDhu6luZyB24bubaSBt4buZdCB0aHXhuq10IHRvw6FuIMSRxrDhu6NjIGNo4buNbi4gDQoNCkvhur90IHF14bqjIGNoaSByYSBy4bqxbmcg4bqjbmggaMaw4bufbmcgY+G7p2EgY8OhYyBwaMawxqFuZyBwaMOhcCBjaOG7jW4gbeG6q3UgbMOqbiBjaOG6pXQgbMaw4bujbmcgcGjDom4gbG/huqFpIGPhu6dhIG3DtCBow6xuaCAodGhlbyB0acOqdSBjaMOtIEFVQykgbMOgIGtow7RuZyBuaOG6pXQgcXXDoW4gduG7m2kgbeG7l2kgbeG7mXQgdGh14bqtdCB0b8OhbiDEkcaw4bujYyBz4butIGThu6VuZyAoaMOsbmggMSk6IA0KDQohW10oQzpcXFVzZXJzXFxBRE1JTlxcRGVza3RvcFxccGljMS5qcGcpDQoNCkvhur90IHF14bqjIG7DoHkgY2jhu4kgcmEgcuG6sW5nLCB2w60gZOG7pSwgduG7m2kgQWRhQm9vc3QgdsOgIFN1cHBvcnQgVmVjdG9yIE1hY2hpbmUgdGjDrCBEb3duc2FwbGluZyBsw6BtIHTEg25nIMSRw6FuZyBr4buDIEFVQyBuaMawbmcgTmFpdmUgQmF5ZXMgQ2xhc3NpZmllciB0aMOsIGfhuqduIG5oxrAgInRyxqEiIHbhu5tpIGPDoWMgcGjGsMahbmcgcGjDoXAgY2jhu41uIG3huqt1LiANCg0KS+G6v3QgcXXhuqMgbsOgeSBsw6AgY8OzIGNow7p0IGtow6FjIGJp4buHdCBzbyB24bubaSBuaOG7r25nIGvhur90IGx14bqtbiBj4bunYSBbV2luLVZlY3RvciBCbG9nXShodHRwOi8vd3d3Lndpbi12ZWN0b3IuY29tL2Jsb2cvMjAxNS8wMi9kb2VzLWJhbGFuY2luZy1jbGFzc2VzLWltcHJvdmUtY2xhc3NpZmllci1wZXJmb3JtYW5jZS8pLiANCg0KUiBDb2RlcyBjaG8ga+G6v3QgcXXhuqMgdGh1IMSRxrDhu6NjIOG7nyBow6xuaCAxOiANCg0KDQpgYGB7ciwgZXZhbD1GQUxTRX0NCg0KIyBMb2FkIHBhY2thZ2VzIGFuZCBDYXJhdmFuIGRhdGEgc2V0OiANCnJtKGxpc3QgPSBscygpKQ0KbGlicmFyeShJU0xSKQ0KbGlicmFyeSh0aWR5dmVyc2UpDQpsaWJyYXJ5KG1hZ3JpdHRyKQ0KbGlicmFyeShjYXJldCkNCmRhdGEoIkNhcmF2YW4iKQ0KDQojIEZ1bmN0aW9uIGZvciBkYXRhIHByb2Nlc3Npbmc6IA0KDQpwcm9jZXNzaW5nRGF0YSA8LSBmdW5jdGlvbiguLi4pIHsNCiAgDQogIFB1cmNoYXNlIDwtIENhcmF2YW4kUHVyY2hhc2UNCiAgZGZfbnVtIDwtIENhcmF2YW4gJT4lIHNlbGVjdCgtUHVyY2hhc2UpDQogIG5lYXJfemVyb3MgPC0gbmVhclplcm9WYXIoZGZfbnVtKQ0KICBkZl9udW1fczEgPC0gZGZfbnVtICU+JSBzZWxlY3QoLW5lYXJfemVyb3MpDQogIGhpZ2hDb3JycyA8LSBmaW5kQ29ycmVsYXRpb24oY29yKGRmX251bV9zMSksIGN1dG9mZiA9IDAuOSkNCiAgZGZfbnVtX3MxICU+JSANCiAgICBzZWxlY3QoLWhpZ2hDb3JycykgJT4lIA0KICAgIG11dGF0ZShQdXJjaGFzZSA9IFB1cmNoYXNlKSAlPiUgDQogICAgcmV0dXJuKCkNCn0NCg0KDQojIFVzZSB0aGUgZnVuY3Rpb246IA0KDQpkZiA8LSBwcm9jZXNzaW5nRGF0YSgpDQoNCiMgU3BsaXQgZGF0YTogDQpzZXQuc2VlZCgxKQ0KaWQgPC0gY3JlYXRlRGF0YVBhcnRpdGlvbih5ID0gZGYkUHVyY2hhc2UsIHAgPSAwLjcsIGxpc3QgPSBGQUxTRSkNCmRmX3RyYWluX21sIDwtIGRmW2lkLCBdDQpkZl90ZXN0X21sIDwtIGRmWy1pZCwgXQ0KDQojIFNldCBjb25kaXRpb25zIGZvciB0cmFpbmluZyBtb2RlbCBhbmQgY3Jvc3MtdmFsaWRhdGlvbjogDQoNCnNldC5zZWVkKDEpDQpudW1iZXIgPC0gMw0KcmVwZWF0cyA8LSA1DQpjb250cm9sIDwtIHRyYWluQ29udHJvbChtZXRob2QgPSAicmVwZWF0ZWRjdiIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgbnVtYmVyID0gbnVtYmVyICwgDQogICAgICAgICAgICAgICAgICAgICAgICByZXBlYXRzID0gcmVwZWF0cywgDQogICAgICAgICAgICAgICAgICAgICAgICBjbGFzc1Byb2JzID0gVFJVRSwgDQogICAgICAgICAgICAgICAgICAgICAgICBzYXZlUHJlZGljdGlvbnMgPSAiZmluYWwiLCANCiAgICAgICAgICAgICAgICAgICAgICAgIGluZGV4ID0gY3JlYXRlUmVzYW1wbGUoZGZfdHJhaW5fbWwkUHVyY2hhc2UsIHJlcGVhdHMqbnVtYmVyKSwgDQogICAgICAgICAgICAgICAgICAgICAgICBzdW1tYXJ5RnVuY3Rpb24gPSBtdWx0aUNsYXNzU3VtbWFyeSwgDQogICAgICAgICAgICAgICAgICAgICAgICBhbGxvd1BhcmFsbGVsID0gVFJVRSkNCg0KDQoNCiMgVXNlIFBhcmFsbGVsIGNvbXB1dGluZzogDQpsaWJyYXJ5KGRvUGFyYWxsZWwpDQpyZWdpc3RlckRvUGFyYWxsZWwoY29yZXMgPSBkZXRlY3RDb3JlcygpIC0gMSkNCg0KIyA2IG1vZGVscyBzZWxlY3RlZDogDQoNCm15X21vZGVscyA8LSBjKCJyZiIsICJhZGFib29zdCIsICJrbm4iLCAic3ZtUmFkaWFsIiwgImdsbSIsICJuYiIpDQoNCiMgVHJhaW4gdGhlc2UgTUwgTW9kZWxzOiANCmxpYnJhcnkoY2FyZXRFbnNlbWJsZSkNCnNldC5zZWVkKDEpDQpzeXN0ZW0udGltZShtb2RlbF9saXN0MSA8LSBjYXJldExpc3QoUHVyY2hhc2Ugfi4sIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBkZl90cmFpbl9tbCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0ckNvbnRyb2wgPSBjb250cm9sLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1ldHJpYyA9ICJBY2N1cmFjeSIsIA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1ldGhvZExpc3QgPSBteV9tb2RlbHMpKQ0KDQoNCiMgRnVuY3Rpb24gZm9yIGNvbGxlY3RpbmcgcmVzdWx0czogDQpnZXRSZXN1bHRzIDwtIGZ1bmN0aW9uKGxpc3RfbW9kZWxzKSB7DQogIA0KICBsaXN0X29mX3Jlc3VsdHMgPC0gbGFwcGx5KG15X21vZGVscywgZnVuY3Rpb24oeCkge2xpc3RfbW9kZWxzW1t4XV0kcmVzYW1wbGV9KQ0KICB0b3RhbF9kZiA8LSBkby5jYWxsKCJiaW5kX3Jvd3MiLCBsaXN0X29mX3Jlc3VsdHMpDQogIHRvdGFsX2RmICU8PiUgbXV0YXRlKE1vZGVsID0gbGFwcGx5KG15X21vZGVscywgZnVuY3Rpb24oeCkge3JlcCh4LCBudW1iZXIqcmVwZWF0cyl9KSAlPiUgdW5saXN0KCkpDQogIHJldHVybih0b3RhbF9kZikNCn0NCg0KDQojIEZ1bmN0aW9uIGZvciBjb21wYXJpbmcgQXZlcmFnZSBBY2N1cmFjeSBiYXNlZCBvbiAxNSBzYW1wbGVzIGZvciB0aGVzZSBtb2RlbHM6IA0KDQpjb21wYXJpbmdBVUMgPC0gZnVuY3Rpb24oZGZfc2VsZWN0ZWQpIHsNCiAgDQogIGRmX3NlbGVjdGVkICU+JSANCiAgICBncm91cF9ieShNb2RlbCkgJT4lIA0KICAgIHN1bW1hcmlzZShhdmdfYWNjID0gbWVhbihBVUMpKSAlPiUgDQogICAgdW5ncm91cCgpICU+JSANCiAgICBhcnJhbmdlKC1hdmdfYWNjKSAlPiUgDQogICAgbXV0YXRlX2lmKGlzLm51bWVyaWMsIGZ1bmN0aW9uKHgpIHtyb3VuZCh4LCAzKX0pICU+JSANCiAgICByZXR1cm4oKQ0KfQ0KDQoNCg0KIyBSZXNhbXBsaW5nIHRlY2huaXF1ZXM6IA0KDQpyZV9zYW1wbGluZyA8LSBjKCJ1cCIsICJkb3duIiwgInNtb3RlIikNCg0KIyBGdW5jdGlvbiBmb3IgdHJhaW5pbmcgbW9kZWwgYnkgcmVzYW1wbGluZyB0ZWNobmlxdWU6IA0KDQpteV90cmFpbkJ5U2FtcGxpbmcgPC0gZnVuY3Rpb24oc2FtcGxpbmcpIHsNCiAgY29udHJvbCRzYW1wbGluZyA8LSBzYW1wbGluZw0KICBtb2RlbF9saXN0IDwtIGNhcmV0TGlzdChQdXJjaGFzZSB+LiwgDQogICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBkZl90cmFpbl9tbCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgdHJDb250cm9sID0gY29udHJvbCwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgbWV0cmljID0gIkFjY3VyYWN5IiwgDQogICAgICAgICAgICAgICAgICAgICAgICAgIG1ldGhvZExpc3QgPSBteV9tb2RlbHMpDQogIHJldHVybihtb2RlbF9saXN0KQ0KfQ0KDQojIFRyYWluIG1vZGVscyBieSByZXNhbXBsaW5nIHRlY2huaXF1ZTogDQoNCmxhcHBseShyZV9zYW1wbGluZywgbXlfdHJhaW5CeVNhbXBsaW5nKSAtPiBhbGxfbW9kZWxzDQoNCg0KIyBDb21wYXJlIEFVQzogDQoNCg0KbW9kZWxfbGlzdDEgJT4lIA0KICBnZXRSZXN1bHRzKCkgJT4lIA0KICBjb21wYXJpbmdBVUMoKSAlPiUgDQogIG11dGF0ZShNZXRob2QgPSAiTm9uZSIpICU+JSANCiAgYmluZF9yb3dzKGFsbF9tb2RlbHNbWzFdXSAlPiUgZ2V0UmVzdWx0cygpICU+JSBjb21wYXJpbmdBVUMoKSAlPiUgbXV0YXRlKE1ldGhvZCA9ICJVcCIpKSAlPiUgDQogIGJpbmRfcm93cyhhbGxfbW9kZWxzW1syXV0gJT4lIGdldFJlc3VsdHMoKSAlPiUgY29tcGFyaW5nQVVDKCkgJT4lIG11dGF0ZShNZXRob2QgPSAiRG93biIpKSAlPiUgDQogIGJpbmRfcm93cyhhbGxfbW9kZWxzW1szXV0gJT4lIGdldFJlc3VsdHMoKSAlPiUgY29tcGFyaW5nQVVDKCkgJT4lIG11dGF0ZShNZXRob2QgPSAiU01PVEUiKSkgLT4gZGZfYXVjX2NvbQ0KDQoNCmRmX2F1Y19jb20gJT4lIA0KICBnZ3Bsb3QoYWVzKE1vZGVsLCBhdmdfYWNjLCBmaWxsID0gTWV0aG9kKSkgKyANCiAgZ2VvbV9jb2wocG9zaXRpb24gPSAiZG9kZ2UiKSArIA0KICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjKCcjZTQxYTFjJywnIzM3N2ViOCcsJyM0ZGFmNGEnLCcjOTg0ZWEzJykpICsgDQogIHRoZW1lX21pbmltYWwoKSArIA0KICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50LCBsaW1pdHMgPSBjKDAsIDAuNzUpLCBicmVha3MgPSBzZXEoMCwgMC43NSwgYnkgPSAwLjEpKSArIA0KICBzY2FsZV94X2Rpc2NyZXRlKGxhYmVscyA9IGMoIkFkYUJvb3N0IiwgIkdMTSIsICJLTk4iLCAiTkIiLCAiUkYiLCAiU1ZNIikpICsgDQogIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IueCA9IGVsZW1lbnRfYmxhbmsoKSkgKyANCiAgbGFicyh4ID0gTlVMTCwgeSA9ICJBdmVyYWdlIEFVQyIsIA0KICAgICAgIHN1YnRpdGxlID0gIkRhdGEgU291cmNlOiBUaGUgSW5zdXJhbmNlIENvbXBhbnkgRGF0YSBQcm92aWRlZCBieSBTZW50aWVudCBNYWNoaW5lIFJlc2VhcmNoIiwgDQogICAgICAgdGl0bGUgPSAiRmlndXJlIDE6IE1vZGVsIFBlcmZvcm1hbmNlIGJ5IFJlc2FtcGxlIE1ldGhvZCBVc2VkIGZvciBTb21lIE1MIEFwcHJvYWNoZXMiKQ0KYGBgDQoNCg==