In this notebook I explore the Human Activity Recognition dataset from the UCI Machnine Learning Repository.

library(plyr)
library(dplyr)
library(xgboost)
library(MLmetrics)
library(ggplot2)
library(grid)
library(gridExtra)

source('prepare_data.R')

Data cleaning and preparation is performed using prepare_data.py. Let’s first visualize the data using the principal components:

X_tr <- X_full %>% filter(is.train == 1) %>% dplyr::select(-is.train) # Select training data
pca_tr <- prcomp(X_tr) # Get principal components
# Plot
dat_pca <- data.frame(pc1 = pca_tr$x[,1], pc2 = pca_tr$x[,2], pc3 = pca_tr$x[,3], label = as.factor(y_train$label))
levels(dat_pca$label) <- c("WALKING", "WALKING.UP", "WALKING.DOWN", "SITTING", "STANDING", "LAYING") g0 <- ggplot(data = dat_pca, aes(pc1, pc2)) + geom_point(aes(color = label)) g0 It looks like just by using the first principal components, one may be able to differenciate between different activitivies (labels). Specifically, WALKING, WALKING.UP and WALKING.DOWN can be classified almost with a linear decision boundary, while SITTING, STANDING and LAYING are more difficult to classify. Of course, we are looking only into the first principal components which explain the most variance in the data. A larger variance in features (which are kinematic variables) means a larger range of motion, so this behavior is expected. If we were to look at principal components with lower variance, we may actually distinguish the last 3 labels better. In fact, looking at the 4th and 5th principal components, we observe # Plot dat_pca <- data.frame(pc3 = pca_tr$x[,3], pc4 = pca_tr$x[,4], pc5 = pca_tr$x[,5],
label = as.factor(y_train$label)) levels(dat_pca$label) <- c("WALKING", "WALKING.UP", "WALKING.DOWN", "SITTING", "STANDING", "LAYING")
g1 <- ggplot(data = dat_pca, aes(pc4, pc5)) + geom_point(aes(color = label))
g1

This looks much better for SITTING, STANDING and LAYING. From this simple investigation, it looks like that linear classifiers may actually do a decent job.

The tree booster have a number of hyperparameters that need to be determined using cross-validation (CV). This is performed in the script train_xgboost.R. Here, we will use the hyperparameters determined from 5-fold CV. First, we convert the training and testing data into matrices that XGBoost likes to use:

library(xgboost)
# Split back into train/test
X_tr <- X_full %>% filter(is.train == 1) %>% dplyr::select(-is.train)
X_tst <- X_full %>% filter(is.train == 0) %>% dplyr::select(-is.train)
y_tr <- as.factor(y_train$label) y_tr <- revalue(y_tr, c('6'='5', '5'='4', '4'='3', '3'='2', '2'='1', '1'='0')) y_tr <- as.numeric(levels(y_tr))[y_tr] y_tst <- as.factor(y_test$label)
y_tst <- revalue(y_tst, c('6'='5', '5'='4', '4'='3', '3'='2', '2'='1', '1'='0'))
y_tst <- as.numeric(levels(y_tst))[y_tst]
# XGB style matrices
dtrain <- xgb.DMatrix(as.matrix(X_tr), label = y_tr)
dtest <- xgb.DMatrix(as.matrix(X_tst), label = y_tst)
watchlist <- list(train=dtrain, test=dtest)

Now, we can fit the tree booster:

# Use best model params
params <- list(booster = "gbtree",
eval_metric = "mlogloss",
objective = "multi:softprob",
eta = 0.50,
max_depth = 2,
gamma = 0.0,
min_child_weight = 0,
colsample_bytree = 0.2,
subsample = 1)
modxgb <- xgb.train(params = params,
data = dtrain,
num_class = 6,
nrounds = 499,
watchlist = watchlist,
verbose = 0) # Change this to 1 to watch the progress

Let’s look at the feature importances:

# Importance matrix
importance_matrix <- xgb.importance(model = modxgb) %>%
mutate(Feature = as.integer(Feature))
top10 <- importance_matrix[1:10, ]$Feature top10 <- paste0("f", top10) # Select top 10 fetures from importance # Names of these features feat_names_10 <- feat_names %>% filter(code %in% top10) %>% mutate(code = substring(code, 2)) %>% mutate(code = as.integer(code)) reord <- match(feat_names_10$code, importance_matrix[1:10,]$Feature) feat_names_10 %>% dplyr::slice(reord) Now, let’s plot the importance as a function of Gain in trees split when a specific feature is used. plt.data <- importance_matrix[1:10, ] plt.data$Feature <- feat_names_10 %>% dplyr::slice(reord) %>% dplyr::select(feature)
xgb.plot.importance(plt.data)

Finally, let’s test the model performance on the test set

# Predict
pred_tst <- predict(modxgb, newdata = dtest)
# Reshape in N x n_class
pred_matrix <- matrix(pred_tst, nrow = nrow(X_tst), byrow = TRUE) # Reshape for class probs
# Accuracy
pred_labels <- apply(pred_matrix, 1, which.max)
cat("Accuracy:", sum(pred_labels == y_test$label) / length(y_test$label), "\n")
Accuracy: 0.955548 
# One-hot encoding and multi-class log loss
expanded_tst <- diag(6)
expanded_tst <- t(expanded_tst[, y_test$label]) cat("MLogLoss: ", mlogloss(pred_matrix, expanded_tst), "\n") MLogLoss: 0.1406249  The confusion matrix provides us information on how well we did for each class: library(caret) confusionMatrix(pred_labels, y_test$label)
Confusion Matrix and Statistics

Reference
Prediction   1   2   3   4   5   6
1 486  26   4   0   0   0
2   9 441  16   2   0   0
3   1   4 400   0   0   0
4   0   0   0 440  20   0
5   0   0   0  49 512   0
6   0   0   0   0   0 537

Overall Statistics

Accuracy : 0.9555
95% CI : (0.9475, 0.9627)
No Information Rate : 0.1822
P-Value [Acc > NIR] : < 2.2e-16

Kappa : 0.9466
Mcnemar's Test P-Value : NA

Statistics by Class:

Class: 1 Class: 2 Class: 3 Class: 4 Class: 5 Class: 6
Sensitivity            0.9798   0.9363   0.9524   0.8961   0.9624   1.0000
Specificity            0.9878   0.9891   0.9980   0.9919   0.9797   1.0000
Pos Pred Value         0.9419   0.9423   0.9877   0.9565   0.9127   1.0000
Neg Pred Value         0.9959   0.9879   0.9921   0.9795   0.9916   1.0000
Prevalence             0.1683   0.1598   0.1425   0.1666   0.1805   0.1822
Detection Rate         0.1649   0.1496   0.1357   0.1493   0.1737   0.1822
Detection Prevalence   0.1751   0.1588   0.1374   0.1561   0.1904   0.1822
Balanced Accuracy      0.9838   0.9627   0.9752   0.9440   0.9711   1.0000

Similar to the tree booster, the linear booster has a bunch of hyperparameters that need to be determined by CV. This is performed in the script train_xgboost_linear.R. Here, we will use the hyperparameters determined from 5-fold CV.

# Use best model params
params <- list(booster = "gblinear",
eval_metric = "mlogloss",
objective = "multi:softprob",
alpha = 0.1,
lambda = 1,
eta = 0.3)
modxgb <- xgb.train(params = params,
data = dtrain,
num_class = 6,
nrounds = 497,
watchlist = watchlist,
verbose = 0) # Change this to 1 to watch the progress

Now predict on the test set

# Predict
pred_tst <- predict(modxgb, newdata = dtest)
# Reshape in N x n_class
pred_matrix <- matrix(pred_tst, nrow = nrow(X_tst), byrow = TRUE) # Reshape for class probs
# Accuracy
pred_labels <- apply(pred_matrix, 1, which.max)
cat("Accuracy:", sum(pred_labels == y_test$label) / length(y_test$label), "\n")
Accuracy: 0.9562267 
# Multi-class log loss
cat("MLogLoss: ", mlogloss(pred_matrix, expanded_tst), "\n")
MLogLoss:  0.1324232 

And the confusion matrix

confusionMatrix(pred_labels, y_test\$label)
Confusion Matrix and Statistics

Reference
Prediction   1   2   3   4   5   6
1 492  27   5   0   1   0
2   0 442  16   2   0   0
3   4   2 398   0   0   0
4   0   0   0 434  12   0
5   0   0   1  54 519   4
6   0   0   0   1   0 533

Overall Statistics

Accuracy : 0.9562
95% CI : (0.9482, 0.9633)
No Information Rate : 0.1822
P-Value [Acc > NIR] : < 2.2e-16

Kappa : 0.9474
Mcnemar's Test P-Value : NA

Statistics by Class:

Class: 1 Class: 2 Class: 3 Class: 4 Class: 5 Class: 6
Sensitivity            0.9919   0.9384   0.9476   0.8839   0.9756   0.9926
Specificity            0.9865   0.9927   0.9976   0.9951   0.9756   0.9996
Pos Pred Value         0.9371   0.9609   0.9851   0.9731   0.8979   0.9981
Neg Pred Value         0.9983   0.9883   0.9913   0.9772   0.9945   0.9983
Prevalence             0.1683   0.1598   0.1425   0.1666   0.1805   0.1822
Detection Rate         0.1669   0.1500   0.1351   0.1473   0.1761   0.1809
Detection Prevalence   0.1781   0.1561   0.1371   0.1513   0.1961   0.1812
Balanced Accuracy      0.9892   0.9656   0.9726   0.9395   0.9756   0.9961

The results are pretty similar for both boosters.

LS0tCnRpdGxlOiAiRXhwbG9yaW5nIHRoZSBIQVIgZGF0YSIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKSW4gdGhpcyBub3RlYm9vayBJIGV4cGxvcmUgdGhlIFtIdW1hbiBBY3Rpdml0eSBSZWNvZ25pdGlvbiBkYXRhc2V0XShodHRwczovL2FyY2hpdmUuaWNzLnVjaS5lZHUvbWwvbWFjaGluZS1sZWFybmluZy1kYXRhYmFzZXMvMDAyNDAvKSBmcm9tIHRoZSBVQ0kgTWFjaG5pbmUgTGVhcm5pbmcgUmVwb3NpdG9yeS4gCgojIyMgTG9hZGluZyBkYXRhCgpgYGB7cn0KbGlicmFyeShwbHlyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHhnYm9vc3QpCmxpYnJhcnkoTUxtZXRyaWNzKQpsaWJyYXJ5KGdncGxvdDIpCmxpYnJhcnkoZ3JpZCkKbGlicmFyeShncmlkRXh0cmEpCgpzb3VyY2UoJ3ByZXBhcmVfZGF0YS5SJykKYGBgCgpEYXRhIGNsZWFuaW5nIGFuZCBwcmVwYXJhdGlvbiBpcyBwZXJmb3JtZWQgdXNpbmcgYHByZXBhcmVfZGF0YS5weWAuIExldCdzIGZpcnN0IHZpc3VhbGl6ZSB0aGUgZGF0YSB1c2luZyB0aGUgcHJpbmNpcGFsIGNvbXBvbmVudHM6CgpgYGB7cn0KWF90ciA8LSBYX2Z1bGwgJT4lIGZpbHRlcihpcy50cmFpbiA9PSAxKSAlPiUgZHBseXI6OnNlbGVjdCgtaXMudHJhaW4pICMgU2VsZWN0IHRyYWluaW5nIGRhdGEKcGNhX3RyIDwtIHByY29tcChYX3RyKSAjIEdldCBwcmluY2lwYWwgY29tcG9uZW50cwoKIyBQbG90ICAKZGF0X3BjYSA8LSBkYXRhLmZyYW1lKHBjMSA9IHBjYV90ciR4WywxXSwgcGMyID0gcGNhX3RyJHhbLDJdLCBwYzMgPSBwY2FfdHIkeFssM10sIAogICAgICAgICAgICAgICAgICAgICAgbGFiZWwgPSBhcy5mYWN0b3IoeV90cmFpbiRsYWJlbCkpCmxldmVscyhkYXRfcGNhJGxhYmVsKSA8LSBjKCJXQUxLSU5HIiwgIldBTEtJTkcuVVAiLCAiV0FMS0lORy5ET1dOIiwgIlNJVFRJTkciLCAiU1RBTkRJTkciLCAiTEFZSU5HIikKCmcwIDwtIGdncGxvdChkYXRhID0gZGF0X3BjYSwgYWVzKHBjMSwgcGMyKSkgKyBnZW9tX3BvaW50KGFlcyhjb2xvciA9IGxhYmVsKSkKZzAKYGBgCiBJdCBsb29rcyBsaWtlIGp1c3QgYnkgdXNpbmcgdGhlIGZpcnN0IHByaW5jaXBhbCBjb21wb25lbnRzLCBvbmUgbWF5IGJlIGFibGUgdG8gZGlmZmVyZW5jaWF0ZSBiZXR3ZWVuIGRpZmZlcmVudCBhY3Rpdml0aXZpZXMgKGxhYmVscykuIFNwZWNpZmljYWxseSwgYFdBTEtJTkdgLCBgV0FMS0lORy5VUGAgYW5kIGBXQUxLSU5HLkRPV05gIGNhbiBiZSBjbGFzc2lmaWVkIGFsbW9zdCB3aXRoIGEgbGluZWFyIGRlY2lzaW9uIGJvdW5kYXJ5LCB3aGlsZSBgU0lUVElOR2AsIGBTVEFORElOR2AgYW5kIGBMQVlJTkdgIGFyZSBtb3JlIGRpZmZpY3VsdCB0byBjbGFzc2lmeS4gT2YgY291cnNlLCB3ZSBhcmUgbG9va2luZyBvbmx5IGludG8gdGhlIGZpcnN0IHByaW5jaXBhbCBjb21wb25lbnRzIHdoaWNoIGV4cGxhaW4gdGhlIG1vc3QgdmFyaWFuY2UgaW4gdGhlIGRhdGEuIEEgbGFyZ2VyIHZhcmlhbmNlIGluIGZlYXR1cmVzICh3aGljaCBhcmUga2luZW1hdGljIHZhcmlhYmxlcykgbWVhbnMgYSBsYXJnZXIgcmFuZ2Ugb2YgbW90aW9uLCBzbyB0aGlzIGJlaGF2aW9yIGlzIGV4cGVjdGVkLiBJZiB3ZSB3ZXJlIHRvIGxvb2sgYXQgcHJpbmNpcGFsIGNvbXBvbmVudHMgd2l0aCBsb3dlciB2YXJpYW5jZSwgd2UgbWF5IGFjdHVhbGx5IGRpc3Rpbmd1aXNoIHRoZSBsYXN0IDMgbGFiZWxzIGJldHRlci4gSW4gZmFjdCwgbG9va2luZyBhdCB0aGUgNHRoIGFuZCA1dGggcHJpbmNpcGFsIGNvbXBvbmVudHMsIHdlIG9ic2VydmUgCiAKYGBge3J9CiMgUGxvdCAgCmRhdF9wY2EgPC0gZGF0YS5mcmFtZShwYzMgPSBwY2FfdHIkeFssM10sIHBjNCA9IHBjYV90ciR4Wyw0XSwgcGM1ID0gcGNhX3RyJHhbLDVdLCAKICAgICAgICAgICAgICAgICAgICAgIGxhYmVsID0gYXMuZmFjdG9yKHlfdHJhaW4kbGFiZWwpKQpsZXZlbHMoZGF0X3BjYSRsYWJlbCkgPC0gYygiV0FMS0lORyIsICJXQUxLSU5HLlVQIiwgIldBTEtJTkcuRE9XTiIsICJTSVRUSU5HIiwgIlNUQU5ESU5HIiwgIkxBWUlORyIpCgpnMSA8LSBnZ3Bsb3QoZGF0YSA9IGRhdF9wY2EsIGFlcyhwYzQsIHBjNSkpICsgZ2VvbV9wb2ludChhZXMoY29sb3IgPSBsYWJlbCkpCmcxCmBgYApUaGlzIGxvb2tzIG11Y2ggYmV0dGVyIGZvciBgU0lUVElOR2AsIGBTVEFORElOR2AgYW5kIGBMQVlJTkdgLiBGcm9tIHRoaXMgc2ltcGxlIGludmVzdGlnYXRpb24sIGl0IGxvb2tzIGxpa2UgdGhhdCBsaW5lYXIgY2xhc3NpZmllcnMgbWF5IGFjdHVhbGx5IGRvIGEgZGVjZW50IGpvYi4gCiAKIyMjIEdyYWRpZW50IEJvb3N0aW5nOiBUcmVlIEJvb3N0ZXIKVGhlIHRyZWUgYm9vc3RlciBoYXZlIGEgbnVtYmVyIG9mIGh5cGVycGFyYW1ldGVycyB0aGF0IG5lZWQgdG8gYmUgZGV0ZXJtaW5lZCB1c2luZyBjcm9zcy12YWxpZGF0aW9uIChDVikuIFRoaXMgaXMgcGVyZm9ybWVkIGluIHRoZSBzY3JpcHQgYHRyYWluX3hnYm9vc3QuUmAuIEhlcmUsIHdlIHdpbGwgdXNlIHRoZSBoeXBlcnBhcmFtZXRlcnMgZGV0ZXJtaW5lZCBmcm9tIDUtZm9sZCBDVi4gRmlyc3QsIHdlIGNvbnZlcnQgdGhlIHRyYWluaW5nIGFuZCB0ZXN0aW5nIGRhdGEgaW50byBtYXRyaWNlcyB0aGF0IFhHQm9vc3QgbGlrZXMgdG8gdXNlOgoKYGBge3J9CmxpYnJhcnkoeGdib29zdCkKIyBTcGxpdCBiYWNrIGludG8gdHJhaW4vdGVzdApYX3RyIDwtIFhfZnVsbCAlPiUgZmlsdGVyKGlzLnRyYWluID09IDEpICU+JSBkcGx5cjo6c2VsZWN0KC1pcy50cmFpbikKWF90c3QgPC0gWF9mdWxsICU+JSBmaWx0ZXIoaXMudHJhaW4gPT0gMCkgJT4lIGRwbHlyOjpzZWxlY3QoLWlzLnRyYWluKQoKIyBYR2Jvb3N0IHdhbnRzIGxldmVscyB0byBzdGFydCB3aXRoIDAKeV90ciA8LSBhcy5mYWN0b3IoeV90cmFpbiRsYWJlbCkKeV90ciA8LSByZXZhbHVlKHlfdHIsIGMoJzYnPSc1JywgJzUnPSc0JywgJzQnPSczJywgJzMnPScyJywgJzInPScxJywgJzEnPScwJykpCnlfdHIgPC0gYXMubnVtZXJpYyhsZXZlbHMoeV90cikpW3lfdHJdCnlfdHN0IDwtIGFzLmZhY3Rvcih5X3Rlc3QkbGFiZWwpCnlfdHN0IDwtIHJldmFsdWUoeV90c3QsIGMoJzYnPSc1JywgJzUnPSc0JywgJzQnPSczJywgJzMnPScyJywgJzInPScxJywgJzEnPScwJykpCnlfdHN0IDwtIGFzLm51bWVyaWMobGV2ZWxzKHlfdHN0KSlbeV90c3RdCgojIFhHQiBzdHlsZSBtYXRyaWNlcwpkdHJhaW4gPC0geGdiLkRNYXRyaXgoYXMubWF0cml4KFhfdHIpLCBsYWJlbCA9IHlfdHIpCmR0ZXN0IDwtIHhnYi5ETWF0cml4KGFzLm1hdHJpeChYX3RzdCksIGxhYmVsID0geV90c3QpCndhdGNobGlzdCA8LSBsaXN0KHRyYWluPWR0cmFpbiwgdGVzdD1kdGVzdCkKYGBgCgpOb3csIHdlIGNhbiBmaXQgdGhlIHRyZWUgYm9vc3RlcjoKYGBge3J9CiMgVXNlIGJlc3QgbW9kZWwgcGFyYW1zCnBhcmFtcyA8LSBsaXN0KGJvb3N0ZXIgPSAiZ2J0cmVlIiwKICAgICAgICAgICAgICAgZXZhbF9tZXRyaWMgPSAibWxvZ2xvc3MiLAogICAgICAgICAgICAgICBvYmplY3RpdmUgPSAibXVsdGk6c29mdHByb2IiLAogICAgICAgICAgICAgICBldGEgPSAwLjUwLCAgICAgCiAgICAgICAgICAgICAgIG1heF9kZXB0aCA9IDIsIAogICAgICAgICAgICAgICBnYW1tYSA9IDAuMCwgICAgCiAgICAgICAgICAgICAgIG1pbl9jaGlsZF93ZWlnaHQgPSAwLCAKICAgICAgICAgICAgICAgY29sc2FtcGxlX2J5dHJlZSA9IDAuMiwKICAgICAgICAgICAgICAgc3Vic2FtcGxlID0gMSkKCm1vZHhnYiA8LSB4Z2IudHJhaW4ocGFyYW1zID0gcGFyYW1zLAogICAgICAgICAgICAgICAgICAgIGRhdGEgPSBkdHJhaW4sCiAgICAgICAgICAgICAgICAgICAgbnVtX2NsYXNzID0gNiwKICAgICAgICAgICAgICAgICAgICBucm91bmRzID0gNDk5LAogICAgICAgICAgICAgICAgICAgIHdhdGNobGlzdCA9IHdhdGNobGlzdCwKICAgICAgICAgICAgICAgICAgICB2ZXJib3NlID0gMCkgIyBDaGFuZ2UgdGhpcyB0byAxIHRvIHdhdGNoIHRoZSBwcm9ncmVzcwpgYGAKTGV0J3MgbG9vayBhdCB0aGUgZmVhdHVyZSBpbXBvcnRhbmNlczoKCmBgYHtyfQojIEltcG9ydGFuY2UgbWF0cml4CmltcG9ydGFuY2VfbWF0cml4IDwtIHhnYi5pbXBvcnRhbmNlKG1vZGVsID0gbW9keGdiKSAlPiUgCiAgbXV0YXRlKEZlYXR1cmUgPSBhcy5pbnRlZ2VyKEZlYXR1cmUpKQp0b3AxMCA8LSBpbXBvcnRhbmNlX21hdHJpeFsxOjEwLCBdJEZlYXR1cmUKdG9wMTAgPC0gcGFzdGUwKCJmIiwgdG9wMTApICAjIFNlbGVjdCB0b3AgMTAgZmV0dXJlcyBmcm9tIGltcG9ydGFuY2UKCiMgTmFtZXMgb2YgdGhlc2UgZmVhdHVyZXMKZmVhdF9uYW1lc18xMCA8LSBmZWF0X25hbWVzICU+JSBmaWx0ZXIoY29kZSAlaW4lIHRvcDEwKSAlPiUgCiAgbXV0YXRlKGNvZGUgPSBzdWJzdHJpbmcoY29kZSwgMikpICU+JQogIG11dGF0ZShjb2RlID0gYXMuaW50ZWdlcihjb2RlKSkKcmVvcmQgPC0gbWF0Y2goZmVhdF9uYW1lc18xMCRjb2RlLCBpbXBvcnRhbmNlX21hdHJpeFsxOjEwLF0kRmVhdHVyZSkKZmVhdF9uYW1lc18xMCAlPiUgZHBseXI6OnNsaWNlKHJlb3JkKQpgYGAKTm93LCBsZXQncyBwbG90IHRoZSBpbXBvcnRhbmNlIGFzIGEgZnVuY3Rpb24gb2YgW0dhaW5dKGh0dHA6Ly94Z2Jvb3N0LnJlYWR0aGVkb2NzLmlvL2VuL2xhdGVzdC9SLXBhY2thZ2UvZGlzY292ZXJZb3VyRGF0YS5odG1sKSBpbiB0cmVlcyBzcGxpdCB3aGVuIGEgc3BlY2lmaWMgZmVhdHVyZSBpcyB1c2VkLiAKCmBgYHtyfQpwbHQuZGF0YSA8LSBpbXBvcnRhbmNlX21hdHJpeFsxOjEwLCBdCnBsdC5kYXRhJEZlYXR1cmUgPC0gZmVhdF9uYW1lc18xMCAlPiUgZHBseXI6OnNsaWNlKHJlb3JkKSAlPiUgZHBseXI6OnNlbGVjdChmZWF0dXJlKQp4Z2IucGxvdC5pbXBvcnRhbmNlKHBsdC5kYXRhKQpgYGAKRmluYWxseSwgbGV0J3MgdGVzdCB0aGUgbW9kZWwgcGVyZm9ybWFuY2Ugb24gdGhlIHRlc3Qgc2V0CgpgYGB7cn0KIyBQcmVkaWN0CnByZWRfdHN0IDwtIHByZWRpY3QobW9keGdiLCBuZXdkYXRhID0gZHRlc3QpCgojIFJlc2hhcGUgaW4gTiB4IG5fY2xhc3MKcHJlZF9tYXRyaXggPC0gbWF0cml4KHByZWRfdHN0LCBucm93ID0gbnJvdyhYX3RzdCksIGJ5cm93ID0gVFJVRSkgIyBSZXNoYXBlIGZvciBjbGFzcyBwcm9icwoKIyBBY2N1cmFjeQpwcmVkX2xhYmVscyA8LSBhcHBseShwcmVkX21hdHJpeCwgMSwgd2hpY2gubWF4KQpjYXQoIkFjY3VyYWN5OiIsIHN1bShwcmVkX2xhYmVscyA9PSB5X3Rlc3QkbGFiZWwpIC8gbGVuZ3RoKHlfdGVzdCRsYWJlbCksICJcbiIpCgojIE9uZS1ob3QgZW5jb2RpbmcgYW5kIG11bHRpLWNsYXNzIGxvZyBsb3NzCmV4cGFuZGVkX3RzdCA8LSBkaWFnKDYpCmV4cGFuZGVkX3RzdCA8LSB0KGV4cGFuZGVkX3RzdFssIHlfdGVzdCRsYWJlbF0pCmNhdCgiTUxvZ0xvc3M6ICIsIG1sb2dsb3NzKHByZWRfbWF0cml4LCBleHBhbmRlZF90c3QpLCAiXG4iKQpgYGAKVGhlIGNvbmZ1c2lvbiBtYXRyaXggcHJvdmlkZXMgdXMgaW5mb3JtYXRpb24gb24gaG93IHdlbGwgd2UgZGlkIGZvciBlYWNoIGNsYXNzOgpgYGB7cn0KbGlicmFyeShjYXJldCkKY29uZnVzaW9uTWF0cml4KHByZWRfbGFiZWxzLCB5X3Rlc3QkbGFiZWwpCmBgYAoKCiMjIyBHcmFkaWVudCBCb29zdGluZzogTGluZWFyIEJvb3N0ZXIKU2ltaWxhciB0byB0aGUgdHJlZSBib29zdGVyLCB0aGUgbGluZWFyIGJvb3N0ZXIgaGFzIGEgYnVuY2ggb2YgaHlwZXJwYXJhbWV0ZXJzIHRoYXQgbmVlZCB0byBiZSBkZXRlcm1pbmVkIGJ5IENWLiBUaGlzIGlzIHBlcmZvcm1lZCBpbiB0aGUgc2NyaXB0IGB0cmFpbl94Z2Jvb3N0X2xpbmVhci5SYC4gSGVyZSwgd2Ugd2lsbCB1c2UgdGhlIGh5cGVycGFyYW1ldGVycyBkZXRlcm1pbmVkIGZyb20gNS1mb2xkIENWLiAKCmBgYHtyfQojIFVzZSBiZXN0IG1vZGVsIHBhcmFtcwpwYXJhbXMgPC0gbGlzdChib29zdGVyID0gImdibGluZWFyIiwKICAgICAgICAgICAgICAgZXZhbF9tZXRyaWMgPSAibWxvZ2xvc3MiLAogICAgICAgICAgICAgICBvYmplY3RpdmUgPSAibXVsdGk6c29mdHByb2IiLAogICAgICAgICAgICAgICBhbHBoYSA9IDAuMSwKICAgICAgICAgICAgICAgbGFtYmRhID0gMSwKICAgICAgICAgICAgICAgZXRhID0gMC4zKQoKbW9keGdiIDwtIHhnYi50cmFpbihwYXJhbXMgPSBwYXJhbXMsCiAgICAgICAgICAgICAgICAgICAgZGF0YSA9IGR0cmFpbiwKICAgICAgICAgICAgICAgICAgICBudW1fY2xhc3MgPSA2LAogICAgICAgICAgICAgICAgICAgIG5yb3VuZHMgPSA0OTcsCiAgICAgICAgICAgICAgICAgICAgd2F0Y2hsaXN0ID0gd2F0Y2hsaXN0LAogICAgICAgICAgICAgICAgICAgIHZlcmJvc2UgPSAwKSAjIENoYW5nZSB0aGlzIHRvIDEgdG8gd2F0Y2ggdGhlIHByb2dyZXNzCmBgYApOb3cgcHJlZGljdCBvbiB0aGUgdGVzdCBzZXQKCmBgYHtyfQojIFByZWRpY3QKcHJlZF90c3QgPC0gcHJlZGljdChtb2R4Z2IsIG5ld2RhdGEgPSBkdGVzdCkKCiMgUmVzaGFwZSBpbiBOIHggbl9jbGFzcwpwcmVkX21hdHJpeCA8LSBtYXRyaXgocHJlZF90c3QsIG5yb3cgPSBucm93KFhfdHN0KSwgYnlyb3cgPSBUUlVFKSAjIFJlc2hhcGUgZm9yIGNsYXNzIHByb2JzCgojIEFjY3VyYWN5CnByZWRfbGFiZWxzIDwtIGFwcGx5KHByZWRfbWF0cml4LCAxLCB3aGljaC5tYXgpCmNhdCgiQWNjdXJhY3k6Iiwgc3VtKHByZWRfbGFiZWxzID09IHlfdGVzdCRsYWJlbCkgLyBsZW5ndGgoeV90ZXN0JGxhYmVsKSwgIlxuIikKCiMgTXVsdGktY2xhc3MgbG9nIGxvc3MKY2F0KCJNTG9nTG9zczogIiwgbWxvZ2xvc3MocHJlZF9tYXRyaXgsIGV4cGFuZGVkX3RzdCksICJcbiIpCmBgYApBbmQgdGhlIGNvbmZ1c2lvbiBtYXRyaXgKYGBge3J9CmNvbmZ1c2lvbk1hdHJpeChwcmVkX2xhYmVscywgeV90ZXN0JGxhYmVsKQpgYGAKVGhlIHJlc3VsdHMgYXJlIHByZXR0eSBzaW1pbGFyIGZvciBib3RoIGJvb3N0ZXJzLiAKCg==