load A dataset with minute-slices already generated

library(caret)
library(readr)
library(dplyr)
library(reshape2)
library(tidyr)
library(catboost)
library(doMC)
suppressPackageStartupMessages(library("optparse"))
registerDoMC(cores=8)
set.seed(01092020)
#set.seed(21092022)
caprino_data<-readr::read_csv("data/caprino_data_sliced.csv",col_types = 
                                cols(
                                  imu_ts = col_double(),
                                  accX = col_double(),
                                  accY = col_double(),
                                  accZ = col_double(),
                                  gyroX = col_double(),
                                  gyroY = col_double(),
                                  gyroZ = col_double(),
                                  gps_hour = col_time(format = ""),
                                  gps_ts = col_double(),
                                  lat = col_double(),
                                  long = col_double(),
                                  alt = col_double(),
                                  activity = col_character(),
                                  id = col_character()
                                )
)
caprino_data

MAIN

Bag of features using kmeans

cluster_features<-c('accX','accY','accZ','gyroX','gyroY','gyroZ')

  minute_slice <-
    as.vector(t(
      caprino_data %>% group_by(id) %>% 
        summarise(n = n()) %>% select(id)
    ))

  trainidx <- sample(1:length(minute_slice), 
                     (length(minute_slice) * 70) / 100.0)
  data_train <- caprino_data %>% 
    filter(id %in% minute_slice[trainidx])
  data_test <- caprino_data %>% 
    filter(id %in% (minute_slice[-trainidx]))
  
  cluster_results <-
    kmeans(
      x = data_train %>% select(cluster_features),
      centers = 24,
      nstart = 5
    )
did not converge in 10 iterationsQuick-TRANSfer stage steps exceeded maximum (= 8097600)Quick-TRANSfer stage steps exceeded maximum (= 8097600)Quick-TRANSfer stage steps exceeded maximum (= 8097600)
  data_train_cluster <-
    cbind(data_train, cluster = cluster_results$cluster) %>% 
    group_by(activity, cluster)
  
  cluster_results_test <-
    kmeans(x = data_test %>% select(cluster_features),
           centers = cluster_results$centers)
Quick-TRANSfer stage steps exceeded maximum (= 3490700)
  data_test_cluster <-
    cbind(data_test, cluster = cluster_results_test$cluster) %>% 
    group_by(activity, cluster)
  
  
  data_train_cluster_labeled <- data_train_cluster   %>%
    group_by(id, cluster) %>% summarise(n = n()) %>%
    reshape2::dcast(id ~ cluster, fill = 0) %>%
    inner_join(data_train_cluster %>% ungroup() %>%
                 select(id, activity)  %>%
                 unique()  ,
               by = "id") %>% filter(!is.na(activity))
Using n as value column: use value.var to override.
  data_test_cluster_labeled <- data_test_cluster   %>% 
    group_by(id, cluster) %>% summarise(n = n()) %>%
    reshape2::dcast(id ~ cluster, fill = 0) %>%
    inner_join(data_test_cluster %>%  ungroup() %>%
                 select(id, activity)  %>%
                 unique() ,
               by = "id")  %>% filter(!is.na(activity))
Using n as value column: use value.var to override.
  
  data_eval_cluster_labeled_idx <- sample(1:nrow(data_train_cluster_labeled), 
                     (length(data_train_cluster_labeled) * 70) / 100.0)
  data_eval_cluster_labeled <- data_train_cluster_labeled[data_eval_cluster_labeled_idx,]
  data_train_cluster_labeled <- data_train_cluster_labeled[-data_eval_cluster_labeled_idx,]

create pool objects as required by catboost R package

pool_train = catboost.load_pool( data_train_cluster_labeled %>% select(-id,-activity), label = as.numeric(as.factor(data_train_cluster_labeled$activity))-1)
pool_test = catboost.load_pool( data_test_cluster_labeled %>% select(-id,-activity), label = as.numeric(as.factor(data_test_cluster_labeled$activity))-1)
pool_valid = catboost.load_pool( data_eval_cluster_labeled %>% select(-id,-activity), label = as.numeric(as.factor(data_eval_cluster_labeled$activity))-1)

Analysis of the CV results

The idea of these analisys was to check overfitting.


cv_results <- catboost.cv(pool_train,fold_count = 3,early_stopping_rounds=40)
Training catboost with default parameters! See help(catboost.train).
0:  learn: 1.5546667    test: 1.5557308 best: 1.5557308 (0) total: 55ms remaining: 54.9s
1:  learn: 1.5246449    test: 1.5258546 best: 1.5258546 (1) total: 99.3ms   remaining: 49.6s
2:  learn: 1.4980554    test: 1.5007952 best: 1.5007952 (2) total: 146ms    remaining: 48.6s
3:  learn: 1.4699261    test: 1.4733930 best: 1.4733930 (3) total: 192ms    remaining: 47.8s
4:  learn: 1.4429316    test: 1.4475796 best: 1.4475796 (4) total: 234ms    remaining: 46.5s
5:  learn: 1.4161196    test: 1.4217049 best: 1.4217049 (5) total: 278ms    remaining: 46s
6:  learn: 1.3923450    test: 1.3993604 best: 1.3993604 (6) total: 323ms    remaining: 45.8s
7:  learn: 1.3689478    test: 1.3756577 best: 1.3756577 (7) total: 365ms    remaining: 45.3s
8:  learn: 1.3467767    test: 1.3554263 best: 1.3554263 (8) total: 405ms    remaining: 44.6s
9:  learn: 1.3230265    test: 1.3337952 best: 1.3337952 (9) total: 449ms    remaining: 44.4s
10: learn: 1.3013922    test: 1.3144407 best: 1.3144407 (10)    total: 489ms    remaining: 44s
11: learn: 1.2787476    test: 1.2927896 best: 1.2927896 (11)    total: 537ms    remaining: 44.2s
12: learn: 1.2593937    test: 1.2743706 best: 1.2743706 (12)    total: 581ms    remaining: 44.1s
13: learn: 1.2408825    test: 1.2565611 best: 1.2565611 (13)    total: 626ms    remaining: 44.1s
14: learn: 1.2210743    test: 1.2376532 best: 1.2376532 (14)    total: 671ms    remaining: 44s
15: learn: 1.2029177    test: 1.2201907 best: 1.2201907 (15)    total: 717ms    remaining: 44.1s
16: learn: 1.1861164    test: 1.2044531 best: 1.2044531 (16)    total: 762ms    remaining: 44.1s
17: learn: 1.1681988    test: 1.1883744 best: 1.1883744 (17)    total: 804ms    remaining: 43.9s
18: learn: 1.1505414    test: 1.1718311 best: 1.1718311 (18)    total: 854ms    remaining: 44.1s
19: learn: 1.1343379    test: 1.1563124 best: 1.1563124 (19)    total: 900ms    remaining: 44.1s
20: learn: 1.1176808    test: 1.1407940 best: 1.1407940 (20)    total: 942ms    remaining: 43.9s
21: learn: 1.1008836    test: 1.1256388 best: 1.1256388 (21)    total: 985ms    remaining: 43.8s
22: learn: 1.0848902    test: 1.1120146 best: 1.1120146 (22)    total: 1.03s    remaining: 43.8s
23: learn: 1.0694730    test: 1.0981464 best: 1.0981464 (23)    total: 1.08s    remaining: 43.8s
24: learn: 1.0558428    test: 1.0852137 best: 1.0852137 (24)    total: 1.12s    remaining: 43.8s
25: learn: 1.0417165    test: 1.0714072 best: 1.0714072 (25)    total: 1.17s    remaining: 43.7s
26: learn: 1.0283326    test: 1.0594737 best: 1.0594737 (26)    total: 1.21s    remaining: 43.6s
27: learn: 1.0139388    test: 1.0462814 best: 1.0462814 (27)    total: 1.25s    remaining: 43.5s
28: learn: 1.0001038    test: 1.0342054 best: 1.0342054 (28)    total: 1.3s remaining: 43.5s
29: learn: 0.9879122    test: 1.0222030 best: 1.0222030 (29)    total: 1.34s    remaining: 43.5s
30: learn: 0.9753210    test: 1.0099780 best: 1.0099780 (30)    total: 1.39s    remaining: 43.6s
31: learn: 0.9628718    test: 0.9981840 best: 0.9981840 (31)    total: 1.44s    remaining: 43.6s
32: learn: 0.9506591    test: 0.9870443 best: 0.9870443 (32)    total: 1.49s    remaining: 43.5s
33: learn: 0.9395606    test: 0.9769147 best: 0.9769147 (33)    total: 1.52s    remaining: 43.3s
34: learn: 0.9274079    test: 0.9657325 best: 0.9657325 (34)    total: 1.56s    remaining: 43.2s
35: learn: 0.9158973    test: 0.9561968 best: 0.9561968 (35)    total: 1.61s    remaining: 43s
36: learn: 0.9054845    test: 0.9470195 best: 0.9470195 (36)    total: 1.65s    remaining: 42.8s
37: learn: 0.8956594    test: 0.9383930 best: 0.9383930 (37)    total: 1.71s    remaining: 43.4s
38: learn: 0.8852732    test: 0.9291796 best: 0.9291796 (38)    total: 1.86s    remaining: 45.9s
39: learn: 0.8753276    test: 0.9203107 best: 0.9203107 (39)    total: 1.91s    remaining: 45.7s
40: learn: 0.8651100    test: 0.9112230 best: 0.9112230 (40)    total: 1.95s    remaining: 45.5s
41: learn: 0.8553714    test: 0.9029190 best: 0.9029190 (41)    total: 1.99s    remaining: 45.3s
42: learn: 0.8465106    test: 0.8959104 best: 0.8959104 (42)    total: 2.03s    remaining: 45.1s
43: learn: 0.8379103    test: 0.8881591 best: 0.8881591 (43)    total: 2.06s    remaining: 44.9s
44: learn: 0.8287389    test: 0.8804756 best: 0.8804756 (44)    total: 2.1s remaining: 44.6s
45: learn: 0.8203387    test: 0.8737965 best: 0.8737965 (45)    total: 2.14s    remaining: 44.4s
46: learn: 0.8109112    test: 0.8658350 best: 0.8658350 (46)    total: 2.18s    remaining: 44.3s
47: learn: 0.8022520    test: 0.8592646 best: 0.8592646 (47)    total: 2.22s    remaining: 44.1s
48: learn: 0.7954667    test: 0.8533875 best: 0.8533875 (48)    total: 2.27s    remaining: 44s
49: learn: 0.7874713    test: 0.8473385 best: 0.8473385 (49)    total: 2.31s    remaining: 43.8s
50: learn: 0.7803970    test: 0.8412851 best: 0.8412851 (50)    total: 2.35s    remaining: 43.7s
51: learn: 0.7733270    test: 0.8355534 best: 0.8355534 (51)    total: 2.39s    remaining: 43.6s
52: learn: 0.7658120    test: 0.8298573 best: 0.8298573 (52)    total: 2.43s    remaining: 43.4s
53: learn: 0.7581702    test: 0.8237281 best: 0.8237281 (53)    total: 2.47s    remaining: 43.3s
54: learn: 0.7514521    test: 0.8185567 best: 0.8185567 (54)    total: 2.51s    remaining: 43.2s
55: learn: 0.7448434    test: 0.8136347 best: 0.8136347 (55)    total: 2.56s    remaining: 43.1s
56: learn: 0.7385664    test: 0.8093230 best: 0.8093230 (56)    total: 2.6s remaining: 43s
57: learn: 0.7324317    test: 0.8046799 best: 0.8046799 (57)    total: 2.64s    remaining: 42.8s
58: learn: 0.7269454    test: 0.8002833 best: 0.8002833 (58)    total: 2.68s    remaining: 42.8s
59: learn: 0.7207612    test: 0.7956899 best: 0.7956899 (59)    total: 2.72s    remaining: 42.6s
60: learn: 0.7154687    test: 0.7917948 best: 0.7917948 (60)    total: 2.76s    remaining: 42.5s
61: learn: 0.7092621    test: 0.7877410 best: 0.7877410 (61)    total: 2.8s remaining: 42.4s
62: learn: 0.7033175    test: 0.7836840 best: 0.7836840 (62)    total: 2.85s    remaining: 42.4s
63: learn: 0.6982511    test: 0.7815011 best: 0.7815011 (63)    total: 2.89s    remaining: 42.3s
64: learn: 0.6937592    test: 0.7779848 best: 0.7779848 (64)    total: 2.93s    remaining: 42.2s
65: learn: 0.6888347    test: 0.7748722 best: 0.7748722 (65)    total: 2.97s    remaining: 42.1s
66: learn: 0.6840832    test: 0.7713597 best: 0.7713597 (66)    total: 3.01s    remaining: 42s
67: learn: 0.6796079    test: 0.7686787 best: 0.7686787 (67)    total: 3.05s    remaining: 41.9s
68: learn: 0.6749387    test: 0.7645457 best: 0.7645457 (68)    total: 3.1s remaining: 41.8s
69: learn: 0.6712005    test: 0.7616558 best: 0.7616558 (69)    total: 3.14s    remaining: 41.7s
70: learn: 0.6663077    test: 0.7587880 best: 0.7587880 (70)    total: 3.18s    remaining: 41.7s
71: learn: 0.6621420    test: 0.7560162 best: 0.7560162 (71)    total: 3.23s    remaining: 41.6s
72: learn: 0.6579743    test: 0.7529883 best: 0.7529883 (72)    total: 3.27s    remaining: 41.5s
73: learn: 0.6542253    test: 0.7500160 best: 0.7500160 (73)    total: 3.31s    remaining: 41.4s
74: learn: 0.6507815    test: 0.7476704 best: 0.7476704 (74)    total: 3.35s    remaining: 41.3s
75: learn: 0.6468295    test: 0.7450161 best: 0.7450161 (75)    total: 3.39s    remaining: 41.3s
76: learn: 0.6423359    test: 0.7419344 best: 0.7419344 (76)    total: 3.44s    remaining: 41.2s
77: learn: 0.6390185    test: 0.7403802 best: 0.7403802 (77)    total: 3.48s    remaining: 41.1s
78: learn: 0.6354592    test: 0.7384963 best: 0.7384963 (78)    total: 3.52s    remaining: 41s
79: learn: 0.6311511    test: 0.7364296 best: 0.7364296 (79)    total: 3.56s    remaining: 41s
80: learn: 0.6273114    test: 0.7344103 best: 0.7344103 (80)    total: 3.6s remaining: 40.9s
81: learn: 0.6238451    test: 0.7318519 best: 0.7318519 (81)    total: 3.64s    remaining: 40.8s
82: learn: 0.6198690    test: 0.7295914 best: 0.7295914 (82)    total: 3.69s    remaining: 40.7s
83: learn: 0.6166182    test: 0.7277429 best: 0.7277429 (83)    total: 3.73s    remaining: 40.6s
84: learn: 0.6134520    test: 0.7254394 best: 0.7254394 (84)    total: 3.77s    remaining: 40.5s
85: learn: 0.6102150    test: 0.7234162 best: 0.7234162 (85)    total: 3.81s    remaining: 40.5s
86: learn: 0.6082069    test: 0.7220676 best: 0.7220676 (86)    total: 3.85s    remaining: 40.4s
87: learn: 0.6060966    test: 0.7207324 best: 0.7207324 (87)    total: 3.89s    remaining: 40.3s
88: learn: 0.6032387    test: 0.7187611 best: 0.7187611 (88)    total: 3.93s    remaining: 40.2s
89: learn: 0.6002091    test: 0.7171721 best: 0.7171721 (89)    total: 3.97s    remaining: 40.2s
90: learn: 0.5971605    test: 0.7156223 best: 0.7156223 (90)    total: 4.01s    remaining: 40.1s
91: learn: 0.5937851    test: 0.7135029 best: 0.7135029 (91)    total: 4.05s    remaining: 40s
92: learn: 0.5906564    test: 0.7113026 best: 0.7113026 (92)    total: 4.09s    remaining: 39.9s
93: learn: 0.5872073    test: 0.7094049 best: 0.7094049 (93)    total: 4.14s    remaining: 39.9s
94: learn: 0.5840294    test: 0.7078468 best: 0.7078468 (94)    total: 4.18s    remaining: 39.8s
95: learn: 0.5811706    test: 0.7069424 best: 0.7069424 (95)    total: 4.22s    remaining: 39.7s
96: learn: 0.5788289    test: 0.7061283 best: 0.7061283 (96)    total: 4.26s    remaining: 39.7s
97: learn: 0.5762715    test: 0.7046239 best: 0.7046239 (97)    total: 4.3s remaining: 39.6s
98: learn: 0.5744380    test: 0.7033061 best: 0.7033061 (98)    total: 4.35s    remaining: 39.6s
99: learn: 0.5718746    test: 0.7019668 best: 0.7019668 (99)    total: 4.39s    remaining: 39.5s
100:    learn: 0.5692172    test: 0.7008803 best: 0.7008803 (100)   total: 4.43s    remaining: 39.4s
101:    learn: 0.5662640    test: 0.6998591 best: 0.6998591 (101)   total: 4.47s    remaining: 39.4s
102:    learn: 0.5635302    test: 0.6983625 best: 0.6983625 (102)   total: 4.51s    remaining: 39.3s
103:    learn: 0.5613365    test: 0.6972678 best: 0.6972678 (103)   total: 4.55s    remaining: 39.2s
104:    learn: 0.5591149    test: 0.6967810 best: 0.6967810 (104)   total: 4.59s    remaining: 39.1s
105:    learn: 0.5567181    test: 0.6959503 best: 0.6959503 (105)   total: 4.63s    remaining: 39s
106:    learn: 0.5545336    test: 0.6949128 best: 0.6949128 (106)   total: 4.67s    remaining: 39s
107:    learn: 0.5528976    test: 0.6940704 best: 0.6940704 (107)   total: 4.71s    remaining: 38.9s
108:    learn: 0.5507342    test: 0.6932395 best: 0.6932395 (108)   total: 4.75s    remaining: 38.8s
109:    learn: 0.5487529    test: 0.6920876 best: 0.6920876 (109)   total: 4.79s    remaining: 38.8s
110:    learn: 0.5461843    test: 0.6909090 best: 0.6909090 (110)   total: 4.83s    remaining: 38.7s
111:    learn: 0.5437666    test: 0.6900401 best: 0.6900401 (111)   total: 4.87s    remaining: 38.6s
112:    learn: 0.5413663    test: 0.6890027 best: 0.6890027 (112)   total: 4.91s    remaining: 38.6s
113:    learn: 0.5394483    test: 0.6879110 best: 0.6879110 (113)   total: 4.95s    remaining: 38.5s
114:    learn: 0.5376187    test: 0.6872955 best: 0.6872955 (114)   total: 4.99s    remaining: 38.4s
115:    learn: 0.5359253    test: 0.6869511 best: 0.6869511 (115)   total: 5.03s    remaining: 38.4s
116:    learn: 0.5338709    test: 0.6862906 best: 0.6862906 (116)   total: 5.08s    remaining: 38.3s
117:    learn: 0.5319207    test: 0.6851787 best: 0.6851787 (117)   total: 5.12s    remaining: 38.3s
118:    learn: 0.5301513    test: 0.6845632 best: 0.6845632 (118)   total: 5.16s    remaining: 38.2s
119:    learn: 0.5284464    test: 0.6846145 best: 0.6845632 (118)   total: 5.2s remaining: 38.2s
120:    learn: 0.5265046    test: 0.6838789 best: 0.6838789 (120)   total: 5.24s    remaining: 38.1s
121:    learn: 0.5249371    test: 0.6833559 best: 0.6833559 (121)   total: 5.28s    remaining: 38s
122:    learn: 0.5232964    test: 0.6828307 best: 0.6828307 (122)   total: 5.32s    remaining: 37.9s
123:    learn: 0.5211780    test: 0.6822418 best: 0.6822418 (123)   total: 5.36s    remaining: 37.9s
124:    learn: 0.5197820    test: 0.6815410 best: 0.6815410 (124)   total: 5.4s remaining: 37.8s
125:    learn: 0.5180500    test: 0.6808611 best: 0.6808611 (125)   total: 5.44s    remaining: 37.7s
126:    learn: 0.5159637    test: 0.6802596 best: 0.6802596 (126)   total: 5.48s    remaining: 37.7s
127:    learn: 0.5145750    test: 0.6800251 best: 0.6800251 (127)   total: 5.52s    remaining: 37.6s
128:    learn: 0.5131270    test: 0.6795061 best: 0.6795061 (128)   total: 5.56s    remaining: 37.6s
129:    learn: 0.5116268    test: 0.6784462 best: 0.6784462 (129)   total: 5.6s remaining: 37.5s
130:    learn: 0.5101399    test: 0.6786165 best: 0.6784462 (129)   total: 5.64s    remaining: 37.4s
131:    learn: 0.5084703    test: 0.6781740 best: 0.6781740 (131)   total: 5.68s    remaining: 37.4s
132:    learn: 0.5063521    test: 0.6776847 best: 0.6776847 (132)   total: 5.72s    remaining: 37.3s
133:    learn: 0.5050157    test: 0.6773883 best: 0.6773883 (133)   total: 5.76s    remaining: 37.3s
134:    learn: 0.5036008    test: 0.6770109 best: 0.6770109 (134)   total: 5.8s remaining: 37.2s
135:    learn: 0.5025140    test: 0.6769943 best: 0.6769943 (135)   total: 5.84s    remaining: 37.1s
136:    learn: 0.5013190    test: 0.6765617 best: 0.6765617 (136)   total: 5.89s    remaining: 37.1s
137:    learn: 0.4998934    test: 0.6760469 best: 0.6760469 (137)   total: 5.93s    remaining: 37s
138:    learn: 0.4986410    test: 0.6757187 best: 0.6757187 (138)   total: 5.97s    remaining: 37s
139:    learn: 0.4968903    test: 0.6751356 best: 0.6751356 (139)   total: 6.01s    remaining: 36.9s
140:    learn: 0.4952099    test: 0.6746050 best: 0.6746050 (140)   total: 6.05s    remaining: 36.8s
141:    learn: 0.4935071    test: 0.6740446 best: 0.6740446 (141)   total: 6.09s    remaining: 36.8s
142:    learn: 0.4925012    test: 0.6738164 best: 0.6738164 (142)   total: 6.13s    remaining: 36.7s
143:    learn: 0.4911841    test: 0.6731198 best: 0.6731198 (143)   total: 6.17s    remaining: 36.7s
144:    learn: 0.4899715    test: 0.6727332 best: 0.6727332 (144)   total: 6.21s    remaining: 36.6s
145:    learn: 0.4889301    test: 0.6725001 best: 0.6725001 (145)   total: 6.25s    remaining: 36.6s
146:    learn: 0.4873780    test: 0.6722568 best: 0.6722568 (146)   total: 6.29s    remaining: 36.5s
147:    learn: 0.4864126    test: 0.6722149 best: 0.6722149 (147)   total: 6.33s    remaining: 36.4s
148:    learn: 0.4850121    test: 0.6720781 best: 0.6720781 (148)   total: 6.37s    remaining: 36.4s
149:    learn: 0.4836356    test: 0.6720807 best: 0.6720781 (148)   total: 6.41s    remaining: 36.3s
150:    learn: 0.4825786    test: 0.6720383 best: 0.6720383 (150)   total: 6.46s    remaining: 36.3s
151:    learn: 0.4817878    test: 0.6715429 best: 0.6715429 (151)   total: 6.5s remaining: 36.2s
152:    learn: 0.4804707    test: 0.6717624 best: 0.6715429 (151)   total: 6.54s    remaining: 36.2s
153:    learn: 0.4793272    test: 0.6714082 best: 0.6714082 (153)   total: 6.58s    remaining: 36.2s
154:    learn: 0.4775134    test: 0.6714860 best: 0.6714082 (153)   total: 6.63s    remaining: 36.1s
155:    learn: 0.4757407    test: 0.6707870 best: 0.6707870 (155)   total: 6.67s    remaining: 36.1s
156:    learn: 0.4745871    test: 0.6708164 best: 0.6707870 (155)   total: 6.71s    remaining: 36s
157:    learn: 0.4737258    test: 0.6706510 best: 0.6706510 (157)   total: 6.75s    remaining: 36s
158:    learn: 0.4728066    test: 0.6702781 best: 0.6702781 (158)   total: 6.8s remaining: 35.9s
159:    learn: 0.4713185    test: 0.6700400 best: 0.6700400 (159)   total: 6.84s    remaining: 35.9s
160:    learn: 0.4704244    test: 0.6700192 best: 0.6700192 (160)   total: 6.88s    remaining: 35.9s
161:    learn: 0.4691785    test: 0.6698231 best: 0.6698231 (161)   total: 6.92s    remaining: 35.8s
162:    learn: 0.4684795    test: 0.6698795 best: 0.6698231 (161)   total: 6.96s    remaining: 35.7s
163:    learn: 0.4678197    test: 0.6697735 best: 0.6697735 (163)   total: 7s   remaining: 35.7s
164:    learn: 0.4666919    test: 0.6694392 best: 0.6694392 (164)   total: 7.04s    remaining: 35.6s
165:    learn: 0.4658437    test: 0.6696341 best: 0.6694392 (164)   total: 7.08s    remaining: 35.6s
166:    learn: 0.4646699    test: 0.6693300 best: 0.6693300 (166)   total: 7.13s    remaining: 35.6s
167:    learn: 0.4633448    test: 0.6694156 best: 0.6693300 (166)   total: 7.17s    remaining: 35.5s
168:    learn: 0.4622246    test: 0.6690310 best: 0.6690310 (168)   total: 7.21s    remaining: 35.4s
169:    learn: 0.4615469    test: 0.6689276 best: 0.6689276 (169)   total: 7.25s    remaining: 35.4s
170:    learn: 0.4607456    test: 0.6687250 best: 0.6687250 (170)   total: 7.29s    remaining: 35.3s
171:    learn: 0.4596009    test: 0.6684813 best: 0.6684813 (171)   total: 7.33s    remaining: 35.3s
172:    learn: 0.4586368    test: 0.6685398 best: 0.6684813 (171)   total: 7.37s    remaining: 35.2s
173:    learn: 0.4578365    test: 0.6685574 best: 0.6684813 (171)   total: 7.42s    remaining: 35.2s
174:    learn: 0.4563237    test: 0.6681324 best: 0.6681324 (174)   total: 7.46s    remaining: 35.2s
175:    learn: 0.4556363    test: 0.6680228 best: 0.6680228 (175)   total: 7.5s remaining: 35.1s
176:    learn: 0.4549517    test: 0.6679841 best: 0.6679841 (176)   total: 7.54s    remaining: 35s
177:    learn: 0.4540533    test: 0.6676784 best: 0.6676784 (177)   total: 7.58s    remaining: 35s
178:    learn: 0.4532682    test: 0.6674889 best: 0.6674889 (178)   total: 7.63s    remaining: 35s
179:    learn: 0.4522742    test: 0.6671672 best: 0.6671672 (179)   total: 7.67s    remaining: 34.9s
180:    learn: 0.4514701    test: 0.6673411 best: 0.6671672 (179)   total: 7.71s    remaining: 34.9s
181:    learn: 0.4506136    test: 0.6672171 best: 0.6671672 (179)   total: 7.74s    remaining: 34.8s
182:    learn: 0.4496457    test: 0.6671973 best: 0.6671672 (179)   total: 7.79s    remaining: 34.8s
183:    learn: 0.4478632    test: 0.6668724 best: 0.6668724 (183)   total: 7.83s    remaining: 34.7s
184:    learn: 0.4469647    test: 0.6669059 best: 0.6668724 (183)   total: 7.87s    remaining: 34.7s
185:    learn: 0.4462463    test: 0.6666449 best: 0.6666449 (185)   total: 7.91s    remaining: 34.6s
186:    learn: 0.4455176    test: 0.6666825 best: 0.6666449 (185)   total: 7.96s    remaining: 34.6s
187:    learn: 0.4448798    test: 0.6666264 best: 0.6666264 (187)   total: 7.99s    remaining: 34.5s
188:    learn: 0.4439402    test: 0.6665348 best: 0.6665348 (188)   total: 8.03s    remaining: 34.5s
189:    learn: 0.4428849    test: 0.6661133 best: 0.6661133 (189)   total: 8.07s    remaining: 34.4s
190:    learn: 0.4420031    test: 0.6662221 best: 0.6661133 (189)   total: 8.11s    remaining: 34.4s
191:    learn: 0.4411102    test: 0.6662488 best: 0.6661133 (189)   total: 8.15s    remaining: 34.3s
192:    learn: 0.4400317    test: 0.6661568 best: 0.6661133 (189)   total: 8.2s remaining: 34.3s
193:    learn: 0.4388651    test: 0.6660374 best: 0.6660374 (193)   total: 8.24s    remaining: 34.2s
194:    learn: 0.4381641    test: 0.6660622 best: 0.6660374 (193)   total: 8.28s    remaining: 34.2s
195:    learn: 0.4371141    test: 0.6656159 best: 0.6656159 (195)   total: 8.32s    remaining: 34.1s
196:    learn: 0.4360972    test: 0.6655833 best: 0.6655833 (196)   total: 8.37s    remaining: 34.1s
197:    learn: 0.4354655    test: 0.6652771 best: 0.6652771 (197)   total: 8.41s    remaining: 34.1s
198:    learn: 0.4346543    test: 0.6651056 best: 0.6651056 (198)   total: 8.45s    remaining: 34s
199:    learn: 0.4339680    test: 0.6650843 best: 0.6650843 (199)   total: 8.49s    remaining: 34s
200:    learn: 0.4333973    test: 0.6648644 best: 0.6648644 (200)   total: 8.53s    remaining: 33.9s
201:    learn: 0.4328697    test: 0.6648258 best: 0.6648258 (201)   total: 8.57s    remaining: 33.9s
202:    learn: 0.4321111    test: 0.6645511 best: 0.6645511 (202)   total: 8.61s    remaining: 33.8s
203:    learn: 0.4312175    test: 0.6647308 best: 0.6645511 (202)   total: 8.65s    remaining: 33.8s
204:    learn: 0.4305560    test: 0.6646748 best: 0.6645511 (202)   total: 8.69s    remaining: 33.7s
205:    learn: 0.4295356    test: 0.6648765 best: 0.6645511 (202)   total: 8.73s    remaining: 33.7s
206:    learn: 0.4292182    test: 0.6649469 best: 0.6645511 (202)   total: 8.78s    remaining: 33.6s
207:    learn: 0.4287898    test: 0.6649007 best: 0.6645511 (202)   total: 8.82s    remaining: 33.6s
208:    learn: 0.4281966    test: 0.6650682 best: 0.6645511 (202)   total: 8.86s    remaining: 33.5s
209:    learn: 0.4270781    test: 0.6648661 best: 0.6645511 (202)   total: 8.9s remaining: 33.5s
210:    learn: 0.4258138    test: 0.6646424 best: 0.6645511 (202)   total: 8.94s    remaining: 33.4s
211:    learn: 0.4249914    test: 0.6646435 best: 0.6645511 (202)   total: 8.98s    remaining: 33.4s
212:    learn: 0.4242867    test: 0.6647489 best: 0.6645511 (202)   total: 9.02s    remaining: 33.3s
213:    learn: 0.4239456    test: 0.6648746 best: 0.6645511 (202)   total: 9.06s    remaining: 33.3s
214:    learn: 0.4225586    test: 0.6649815 best: 0.6645511 (202)   total: 9.1s remaining: 33.2s
215:    learn: 0.4220276    test: 0.6651206 best: 0.6645511 (202)   total: 9.14s    remaining: 33.2s
216:    learn: 0.4209415    test: 0.6649260 best: 0.6645511 (202)   total: 9.18s    remaining: 33.1s
217:    learn: 0.4203122    test: 0.6649432 best: 0.6645511 (202)   total: 9.23s    remaining: 33.1s
218:    learn: 0.4200476    test: 0.6649174 best: 0.6645511 (202)   total: 9.27s    remaining: 33s
219:    learn: 0.4190111    test: 0.6647229 best: 0.6645511 (202)   total: 9.31s    remaining: 33s
220:    learn: 0.4185022    test: 0.6646524 best: 0.6645511 (202)   total: 9.35s    remaining: 33s
221:    learn: 0.4173976    test: 0.6647194 best: 0.6645511 (202)   total: 9.39s    remaining: 32.9s
222:    learn: 0.4168957    test: 0.6647572 best: 0.6645511 (202)   total: 9.43s    remaining: 32.8s
223:    learn: 0.4164720    test: 0.6647148 best: 0.6645511 (202)   total: 9.47s    remaining: 32.8s
224:    learn: 0.4161214    test: 0.6646966 best: 0.6645511 (202)   total: 9.51s    remaining: 32.7s
225:    learn: 0.4152978    test: 0.6645121 best: 0.6645121 (225)   total: 9.55s    remaining: 32.7s
226:    learn: 0.4141968    test: 0.6641136 best: 0.6641136 (226)   total: 9.59s    remaining: 32.7s
227:    learn: 0.4133075    test: 0.6639965 best: 0.6639965 (227)   total: 9.63s    remaining: 32.6s
228:    learn: 0.4128282    test: 0.6640136 best: 0.6639965 (227)   total: 9.67s    remaining: 32.5s
229:    learn: 0.4121788    test: 0.6639160 best: 0.6639160 (229)   total: 9.71s    remaining: 32.5s
230:    learn: 0.4113296    test: 0.6639310 best: 0.6639160 (229)   total: 9.75s    remaining: 32.5s
231:    learn: 0.4105195    test: 0.6638606 best: 0.6638606 (231)   total: 9.79s    remaining: 32.4s
232:    learn: 0.4103082    test: 0.6638920 best: 0.6638606 (231)   total: 9.83s    remaining: 32.4s
233:    learn: 0.4093325    test: 0.6636613 best: 0.6636613 (233)   total: 9.87s    remaining: 32.3s
234:    learn: 0.4082214    test: 0.6633359 best: 0.6633359 (234)   total: 9.91s    remaining: 32.3s
235:    learn: 0.4074059    test: 0.6634655 best: 0.6633359 (234)   total: 9.95s    remaining: 32.2s
236:    learn: 0.4066522    test: 0.6631987 best: 0.6631987 (236)   total: 9.99s    remaining: 32.2s
237:    learn: 0.4063183    test: 0.6631494 best: 0.6631494 (237)   total: 10s  remaining: 32.1s
238:    learn: 0.4055192    test: 0.6632692 best: 0.6631494 (237)   total: 10.1s    remaining: 32.1s
239:    learn: 0.4052723    test: 0.6633225 best: 0.6631494 (237)   total: 10.1s    remaining: 32s
240:    learn: 0.4045681    test: 0.6631851 best: 0.6631494 (237)   total: 10.2s    remaining: 32s
241:    learn: 0.4040074    test: 0.6633157 best: 0.6631494 (237)   total: 10.2s    remaining: 31.9s
242:    learn: 0.4035004    test: 0.6634353 best: 0.6631494 (237)   total: 10.2s    remaining: 31.9s
243:    learn: 0.4029208    test: 0.6632730 best: 0.6631494 (237)   total: 10.3s    remaining: 31.8s
244:    learn: 0.4021980    test: 0.6632020 best: 0.6631494 (237)   total: 10.3s    remaining: 31.8s
245:    learn: 0.4019256    test: 0.6632761 best: 0.6631494 (237)   total: 10.4s    remaining: 31.7s
246:    learn: 0.4010695    test: 0.6633418 best: 0.6631494 (237)   total: 10.4s    remaining: 31.7s
247:    learn: 0.4000982    test: 0.6631508 best: 0.6631494 (237)   total: 10.4s    remaining: 31.7s
248:    learn: 0.3988271    test: 0.6628454 best: 0.6628454 (248)   total: 10.5s    remaining: 31.6s
249:    learn: 0.3982140    test: 0.6628108 best: 0.6628108 (249)   total: 10.5s    remaining: 31.6s
250:    learn: 0.3978801    test: 0.6628702 best: 0.6628108 (249)   total: 10.6s    remaining: 31.5s
251:    learn: 0.3969788    test: 0.6627177 best: 0.6627177 (251)   total: 10.6s    remaining: 31.5s
252:    learn: 0.3964493    test: 0.6629592 best: 0.6627177 (251)   total: 10.7s    remaining: 31.4s
253:    learn: 0.3959160    test: 0.6629898 best: 0.6627177 (251)   total: 10.7s    remaining: 31.4s
254:    learn: 0.3947624    test: 0.6627030 best: 0.6627030 (254)   total: 10.7s    remaining: 31.3s
255:    learn: 0.3945055    test: 0.6627915 best: 0.6627030 (254)   total: 10.8s    remaining: 31.3s
256:    learn: 0.3935376    test: 0.6626342 best: 0.6626342 (256)   total: 10.8s    remaining: 31.3s
257:    learn: 0.3921622    test: 0.6626160 best: 0.6626160 (257)   total: 10.9s    remaining: 31.2s
258:    learn: 0.3910978    test: 0.6625222 best: 0.6625222 (258)   total: 10.9s    remaining: 31.2s
259:    learn: 0.3901118    test: 0.6624112 best: 0.6624112 (259)   total: 10.9s    remaining: 31.1s
260:    learn: 0.3898942    test: 0.6624649 best: 0.6624112 (259)   total: 11s  remaining: 31.1s
261:    learn: 0.3896687    test: 0.6624553 best: 0.6624112 (259)   total: 11s  remaining: 31s
262:    learn: 0.3894329    test: 0.6624749 best: 0.6624112 (259)   total: 11.1s    remaining: 31s
263:    learn: 0.3890761    test: 0.6624714 best: 0.6624112 (259)   total: 11.1s    remaining: 30.9s
264:    learn: 0.3876030    test: 0.6621739 best: 0.6621739 (264)   total: 11.1s    remaining: 30.9s
265:    learn: 0.3866984    test: 0.6620423 best: 0.6620423 (265)   total: 11.2s    remaining: 30.8s
266:    learn: 0.3861041    test: 0.6618922 best: 0.6618922 (266)   total: 11.2s    remaining: 30.8s
267:    learn: 0.3857912    test: 0.6618734 best: 0.6618734 (267)   total: 11.3s    remaining: 30.7s
268:    learn: 0.3851773    test: 0.6616269 best: 0.6616269 (268)   total: 11.3s    remaining: 30.7s
269:    learn: 0.3849308    test: 0.6617077 best: 0.6616269 (268)   total: 11.3s    remaining: 30.7s
270:    learn: 0.3840345    test: 0.6615913 best: 0.6615913 (270)   total: 11.4s    remaining: 30.6s
271:    learn: 0.3829210    test: 0.6614705 best: 0.6614705 (271)   total: 11.4s    remaining: 30.6s
272:    learn: 0.3821836    test: 0.6615558 best: 0.6614705 (271)   total: 11.5s    remaining: 30.5s
273:    learn: 0.3815772    test: 0.6616284 best: 0.6614705 (271)   total: 11.5s    remaining: 30.5s
274:    learn: 0.3813296    test: 0.6616873 best: 0.6614705 (271)   total: 11.5s    remaining: 30.4s
275:    learn: 0.3808688    test: 0.6617554 best: 0.6614705 (271)   total: 11.6s    remaining: 30.4s
276:    learn: 0.3802471    test: 0.6619092 best: 0.6614705 (271)   total: 11.7s    remaining: 30.6s
277:    learn: 0.3795749    test: 0.6620626 best: 0.6614705 (271)   total: 11.8s    remaining: 30.5s
278:    learn: 0.3783672    test: 0.6619221 best: 0.6614705 (271)   total: 11.8s    remaining: 30.5s
279:    learn: 0.3781360    test: 0.6619358 best: 0.6614705 (271)   total: 11.8s    remaining: 30.4s
280:    learn: 0.3769634    test: 0.6618317 best: 0.6614705 (271)   total: 11.9s    remaining: 30.4s
281:    learn: 0.3759358    test: 0.6619058 best: 0.6614705 (271)   total: 11.9s    remaining: 30.4s
282:    learn: 0.3752704    test: 0.6621846 best: 0.6614705 (271)   total: 12s  remaining: 30.3s
283:    learn: 0.3742622    test: 0.6622451 best: 0.6614705 (271)   total: 12s  remaining: 30.3s
284:    learn: 0.3740157    test: 0.6623301 best: 0.6614705 (271)   total: 12s  remaining: 30.2s
285:    learn: 0.3735616    test: 0.6624327 best: 0.6614705 (271)   total: 12.1s    remaining: 30.2s
286:    learn: 0.3728352    test: 0.6625006 best: 0.6614705 (271)   total: 12.1s    remaining: 30.1s
287:    learn: 0.3720285    test: 0.6624332 best: 0.6614705 (271)   total: 12.2s    remaining: 30.1s
288:    learn: 0.3713876    test: 0.6625495 best: 0.6614705 (271)   total: 12.2s    remaining: 30s
289:    learn: 0.3705413    test: 0.6625185 best: 0.6614705 (271)   total: 12.2s    remaining: 30s
290:    learn: 0.3692390    test: 0.6625825 best: 0.6614705 (271)   total: 12.3s    remaining: 29.9s
291:    learn: 0.3683846    test: 0.6624959 best: 0.6614705 (271)   total: 12.3s    remaining: 29.9s
292:    learn: 0.3679680    test: 0.6624607 best: 0.6614705 (271)   total: 12.4s    remaining: 29.8s
293:    learn: 0.3672124    test: 0.6622991 best: 0.6614705 (271)   total: 12.4s    remaining: 29.8s
294:    learn: 0.3658497    test: 0.6623406 best: 0.6614705 (271)   total: 12.5s    remaining: 29.8s
295:    learn: 0.3649355    test: 0.6626116 best: 0.6614705 (271)   total: 12.5s    remaining: 29.7s
296:    learn: 0.3636890    test: 0.6622750 best: 0.6614705 (271)   total: 12.5s    remaining: 29.7s
297:    learn: 0.3629630    test: 0.6622840 best: 0.6614705 (271)   total: 12.6s    remaining: 29.6s
298:    learn: 0.3623670    test: 0.6621594 best: 0.6614705 (271)   total: 12.6s    remaining: 29.6s
299:    learn: 0.3618878    test: 0.6621685 best: 0.6614705 (271)   total: 12.7s    remaining: 29.5s
300:    learn: 0.3611933    test: 0.6620117 best: 0.6614705 (271)   total: 12.7s    remaining: 29.5s
301:    learn: 0.3602050    test: 0.6622119 best: 0.6614705 (271)   total: 12.7s    remaining: 29.4s
302:    learn: 0.3595735    test: 0.6621643 best: 0.6614705 (271)   total: 12.8s    remaining: 29.4s
303:    learn: 0.3587344    test: 0.6620436 best: 0.6614705 (271)   total: 12.8s    remaining: 29.4s
304:    learn: 0.3576039    test: 0.6620792 best: 0.6614705 (271)   total: 12.9s    remaining: 29.3s
305:    learn: 0.3567728    test: 0.6618119 best: 0.6614705 (271)   total: 12.9s    remaining: 29.3s
306:    learn: 0.3565550    test: 0.6618601 best: 0.6614705 (271)   total: 12.9s    remaining: 29.2s
307:    learn: 0.3558307    test: 0.6620632 best: 0.6614705 (271)   total: 13s  remaining: 29.2s
308:    learn: 0.3547206    test: 0.6621373 best: 0.6614705 (271)   total: 13s  remaining: 29.1s
309:    learn: 0.3539099    test: 0.6623453 best: 0.6614705 (271)   total: 13.1s    remaining: 29.1s
310:    learn: 0.3533895    test: 0.6623720 best: 0.6614705 (271)   total: 13.1s    remaining: 29s
Stopped by overfitting detector  (40 iterations wait)
311:    learn: 0.3525582    test: 0.6624124 best: 0.6614705 (271)   total: 13.1s    remaining: 29s
cv_results %>% select(test.RMSE.mean, train.RMSE.mean) %>% tibble::add_column(id=seq(1:nrow(cv_results))) %>% reshape2::melt(id.var="id") %>%
ggplot(aes(x=id,y=value))+
   geom_line(aes(color=variable,linetype=variable))+
   ggdark::dark_theme_gray()

Training the model

A large iteration but we use early stoping

fit_params <- list(iterations = 1000,
                   loss_function = 'MultiClass',
                   #border_count = 32,
                   depth = 3,
                   #learning_rate = 0.03,
                   #l2_leaf_reg = 3.5,
                   task_type = 'CPU',
                   verbose = 10,
                   early_stopping_rounds=40
                   )
model <- catboost.train(pool_train, pool_valid, params = fit_params)
Learning rate set to 0.106203
0:  learn: 1.3009113    test: 1.2966790 best: 1.2966790 (0) total: 1.66ms   remaining: 1.66s
10: learn: 0.8909315    test: 0.7817814 best: 0.7817814 (10)    total: 13.2ms   remaining: 1.19s
20: learn: 0.7336459    test: 0.6398310 best: 0.6398310 (20)    total: 21.6ms   remaining: 1.01s
30: learn: 0.6483809    test: 0.5784817 best: 0.5784817 (30)    total: 31.9ms   remaining: 996ms
40: learn: 0.5982220    test: 0.5468035 best: 0.5468035 (40)    total: 40.7ms   remaining: 951ms
50: learn: 0.5606395    test: 0.5278374 best: 0.5278374 (50)    total: 52.8ms   remaining: 982ms
60: learn: 0.5342883    test: 0.5173544 best: 0.5173544 (60)    total: 64.2ms   remaining: 988ms
70: learn: 0.5129109    test: 0.4895823 best: 0.4893849 (69)    total: 78.3ms   remaining: 1.02s
80: learn: 0.4932392    test: 0.4867612 best: 0.4867612 (80)    total: 87.2ms   remaining: 989ms
90: learn: 0.4754259    test: 0.4780816 best: 0.4780816 (90)    total: 97.6ms   remaining: 975ms
100:    learn: 0.4601673    test: 0.4660612 best: 0.4660612 (100)   total: 106ms    remaining: 939ms
110:    learn: 0.4436737    test: 0.4515401 best: 0.4515146 (109)   total: 117ms    remaining: 937ms
120:    learn: 0.4253618    test: 0.4281589 best: 0.4281589 (120)   total: 125ms    remaining: 909ms
130:    learn: 0.4194772    test: 0.4305671 best: 0.4275898 (125)   total: 137ms    remaining: 910ms
140:    learn: 0.4054616    test: 0.4298915 best: 0.4275898 (125)   total: 147ms    remaining: 893ms
150:    learn: 0.3978960    test: 0.4309293 best: 0.4275898 (125)   total: 158ms    remaining: 890ms
160:    learn: 0.3864554    test: 0.4297152 best: 0.4275898 (125)   total: 169ms    remaining: 881ms
170:    learn: 0.3751811    test: 0.4194522 best: 0.4181774 (168)   total: 178ms    remaining: 864ms
180:    learn: 0.3663357    test: 0.4180910 best: 0.4164844 (178)   total: 187ms    remaining: 847ms
190:    learn: 0.3585514    test: 0.4063765 best: 0.4060892 (189)   total: 196ms    remaining: 832ms
200:    learn: 0.3530526    test: 0.4031559 best: 0.4031559 (200)   total: 207ms    remaining: 825ms
210:    learn: 0.3465621    test: 0.4017483 best: 0.4014957 (209)   total: 219ms    remaining: 819ms
220:    learn: 0.3424457    test: 0.4019696 best: 0.3995103 (215)   total: 229ms    remaining: 807ms
230:    learn: 0.3363199    test: 0.3966621 best: 0.3965031 (229)   total: 238ms    remaining: 793ms
240:    learn: 0.3275563    test: 0.3929800 best: 0.3929800 (240)   total: 249ms    remaining: 783ms
250:    learn: 0.3198824    test: 0.3939049 best: 0.3914459 (249)   total: 259ms    remaining: 773ms
260:    learn: 0.3160815    test: 0.3924385 best: 0.3914459 (249)   total: 272ms    remaining: 771ms
270:    learn: 0.3116538    test: 0.3892632 best: 0.3889103 (269)   total: 282ms    remaining: 758ms
280:    learn: 0.3064914    test: 0.3859660 best: 0.3859660 (280)   total: 300ms    remaining: 767ms
290:    learn: 0.2999560    test: 0.3867871 best: 0.3852553 (284)   total: 308ms    remaining: 751ms
300:    learn: 0.2953233    test: 0.3860416 best: 0.3852553 (284)   total: 318ms    remaining: 739ms
310:    learn: 0.2921905    test: 0.3848209 best: 0.3848209 (310)   total: 328ms    remaining: 727ms
320:    learn: 0.2844603    test: 0.3932209 best: 0.3847087 (313)   total: 340ms    remaining: 719ms
330:    learn: 0.2793579    test: 0.3912462 best: 0.3847087 (313)   total: 349ms    remaining: 706ms
340:    learn: 0.2739184    test: 0.3900825 best: 0.3847087 (313)   total: 360ms    remaining: 696ms
350:    learn: 0.2682450    test: 0.3920053 best: 0.3847087 (313)   total: 370ms    remaining: 685ms
Stopped by overfitting detector  (40 iterations wait)

bestTest = 0.3847086679
bestIteration = 313

Shrink model to first 314 iterations.

Check the relation between the labels used in RF and the labels used in catboost

as.factor(data_test_cluster_labeled$activity)
  [1] RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP RP W  W  W  W  W  W  W 
 [39] W  W  W  W  W  W  W  G  G  G  G  G  G  G  G  RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF
 [77] RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF RF
[115] RF RF RF RF RF RF RF RF RF RF RF RF G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G 
[153] G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  G  W  W  W  W  W  W  W  W  W  W  W 
[191] W  W  RP RP RP RP RP RP
Levels: G RF RP W
as.factor(as.numeric(as.factor(data_test_cluster_labeled$activity))-1)
  [1] 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 0 0 0 0 0 0 0 0 1 1 1 1
 [58] 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1
[115] 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
[172] 0 0 0 0 0 0 0 0 3 3 3 3 3 3 3 3 3 3 3 3 3 2 2 2 2 2 2
Levels: 0 1 2 3

Confusion Matrix and macro and micro metrics

  data_test_prediction <-catboost.predict(model, pool_test,prediction_type='Class')
  cm <-
    caret::confusionMatrix(as.factor(data_test_prediction),as.factor(as.numeric(as.factor(data_test_cluster_labeled$activity))-1),
                           mode = 'everything')

cm
Confusion Matrix and Statistics

          Reference
Prediction  0  1  2  3
         0 49 20  6  1
         1 10 49  1  0
         2  0  4 29  0
         3  2  0  1 26

Overall Statistics
                                         
               Accuracy : 0.7727         
                 95% CI : (0.708, 0.8291)
    No Information Rate : 0.3687         
    P-Value [Acc > NIR] : < 2.2e-16      
                                         
                  Kappa : 0.6839         
                                         
 Mcnemar's Test P-Value : NA             

Statistics by Class:

                     Class: 0 Class: 1 Class: 2 Class: 3
Sensitivity            0.8033   0.6712   0.7838   0.9630
Specificity            0.8029   0.9120   0.9752   0.9825
Pos Pred Value         0.6447   0.8167   0.8788   0.8966
Neg Pred Value         0.9016   0.8261   0.9515   0.9941
Precision              0.6447   0.8167   0.8788   0.8966
Recall                 0.8033   0.6712   0.7838   0.9630
F1                     0.7153   0.7368   0.8286   0.9286
Prevalence             0.3081   0.3687   0.1869   0.1364
Detection Rate         0.2475   0.2475   0.1465   0.1313
Detection Prevalence   0.3838   0.3030   0.1667   0.1465
Balanced Accuracy      0.8031   0.7916   0.8795   0.9727
  macro_metrics <- (apply(cm$byClass, 2, sum) / 4) [c(5, 6, 7)]
  s <-
    table(data_test_prediction,
          as.factor(data_test_cluster_labeled$activity))
  
  Precision_micro <-  (sum(diag(s)) / sum(apply(s, 1, sum)))
  Recall_micro <- (sum(diag(s)) / sum(apply(s, 2, sum)))
  
  micro_metrics <- c((sum(diag(s)) / sum(apply(s, 1, sum))),
                     (sum(diag(s)) / sum(apply(s, 2, sum))),
                     2 * ((Precision_micro * Recall_micro) / (Precision_micro + Recall_micro)
                     ))
  
  mm <- cbind(c(macro_metrics), c(micro_metrics))
mm
               [,1]      [,2]
Precision 0.8091858 0.7727273
Recall    0.8053146 0.7727273
F1        0.8023284 0.7727273

Confusion Matrix (plot)

len_data_test_prediction = length(data_test_prediction)
reshape2::melt(table(data_test_prediction,as.factor(data_test_cluster_labeled$activity))) %>% mutate(value=(value)) %>%
  ggplot(aes(x=data_test_prediction,y=Var2))+
  geom_tile(aes(fill=value), colour = "white") + 
   geom_text(aes(label = sprintf("%1.2f", value)), vjust = 1)+
  scale_fill_gradient(low = "white", high = "red")+
  xlab(" Predicted Activity ")+ylab(" Actual Activity")+
  theme_bw()+ theme(legend.position = "none")

sessionInfo()
R version 3.6.3 (2020-02-29)
Platform: x86_64-pc-linux-gnu (64-bit)
Running under: Ubuntu 16.04.6 LTS

Matrix products: default
BLAS:   /usr/lib/atlas-base/atlas/libblas.so.3.0
LAPACK: /usr/lib/atlas-base/atlas/liblapack.so.3.0

locale:
 [1] LC_CTYPE=es_AR.UTF-8       LC_NUMERIC=C               LC_TIME=es_AR.UTF-8        LC_COLLATE=es_AR.UTF-8    
 [5] LC_MONETARY=es_AR.UTF-8    LC_MESSAGES=es_AR.UTF-8    LC_PAPER=es_AR.UTF-8       LC_NAME=C                 
 [9] LC_ADDRESS=C               LC_TELEPHONE=C             LC_MEASUREMENT=es_AR.UTF-8 LC_IDENTIFICATION=C       

attached base packages:
[1] parallel  stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] optparse_1.6.4   doMC_1.3.6       iterators_1.0.12 foreach_1.4.7    catboost_0.24.1  tidyr_1.0.0     
 [7] reshape2_1.4.3   dplyr_0.8.3      readr_1.3.1      caret_6.0-84     ggplot2_3.2.1    lattice_0.20-38 
[13] reticulate_1.16 

loaded via a namespace (and not attached):
 [1] Rcpp_1.0.3          lubridate_1.7.4     class_7.3-17        assertthat_0.2.1    digest_0.6.23      
 [6] ipred_0.9-9         R6_2.4.1            plyr_1.8.4          ggdark_0.2.1        backports_1.1.5    
[11] stats4_3.6.3        evaluate_0.14       e1071_1.7-2         pillar_1.4.6        rlang_0.4.7        
[16] lazyeval_0.2.2      rstudioapi_0.10     data.table_1.12.4   rpart_4.1-15        Matrix_1.2-17      
[21] rmarkdown_2.3       labeling_0.3        splines_3.6.3       gower_0.2.1         stringr_1.4.0      
[26] tinytex_0.16        munsell_0.5.0       broom_0.7.0         compiler_3.6.3      xfun_0.10          
[31] pkgconfig_2.0.3     base64enc_0.1-3     htmltools_0.4.0     nnet_7.3-14         tidyselect_0.2.5   
[36] gridExtra_2.3       tibble_3.0.3        prodlim_2018.04.18  codetools_0.2-16    randomForest_4.6-14
[41] fansi_0.4.0         crayon_1.3.4        withr_2.1.2         MASS_7.3-51.6       recipes_0.1.7      
[46] rappdirs_0.3.1      ModelMetrics_1.2.2  grid_3.6.3          nlme_3.1-147        jsonlite_1.6       
[51] gtable_0.3.0        lifecycle_0.2.0     magrittr_1.5        scales_1.1.0        cli_2.0.0          
[56] stringi_1.4.3       farver_2.0.1        getopt_1.20.3       timeDate_3043.102   ellipsis_0.3.0     
[61] generics_0.0.2      vctrs_0.3.2         lava_1.6.6          tools_3.6.3         glue_1.3.1         
[66] purrr_0.3.3         hms_0.5.1           rsconnect_0.8.15    survival_3.1-12     yaml_2.2.0         
[71] colorspace_1.4-1    knitr_1.25         
LS0tCnRpdGxlOiAiVGVzdGluZyBjYXRib29zdCBmb3IgR29hdCBCZWhhdmlvciBjbGFzc2lmaWNhdGlvbiIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQojIyMgbG9hZCBBIGRhdGFzZXQgd2l0aCBtaW51dGUtc2xpY2VzIGFscmVhZHkgZ2VuZXJhdGVkCmBgYHtyfQpsaWJyYXJ5KGNhcmV0KQpsaWJyYXJ5KHJlYWRyKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHJlc2hhcGUyKQpsaWJyYXJ5KHRpZHlyKQpsaWJyYXJ5KGNhdGJvb3N0KQpsaWJyYXJ5KGRvTUMpCnN1cHByZXNzUGFja2FnZVN0YXJ0dXBNZXNzYWdlcyhsaWJyYXJ5KCJvcHRwYXJzZSIpKQpyZWdpc3RlckRvTUMoY29yZXM9OCkKc2V0LnNlZWQoMDEwOTIwMjApCiNzZXQuc2VlZCgyMTA5MjAyMikKY2Fwcmlub19kYXRhPC1yZWFkcjo6cmVhZF9jc3YoImRhdGEvY2Fwcmlub19kYXRhX3NsaWNlZC5jc3YiLGNvbF90eXBlcyA9IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbHMoCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpbXVfdHMgPSBjb2xfZG91YmxlKCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhY2NYID0gY29sX2RvdWJsZSgpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYWNjWSA9IGNvbF9kb3VibGUoKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFjY1ogPSBjb2xfZG91YmxlKCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBneXJvWCA9IGNvbF9kb3VibGUoKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGd5cm9ZID0gY29sX2RvdWJsZSgpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZ3lyb1ogPSBjb2xfZG91YmxlKCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBncHNfaG91ciA9IGNvbF90aW1lKGZvcm1hdCA9ICIiKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGdwc190cyA9IGNvbF9kb3VibGUoKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhdCA9IGNvbF9kb3VibGUoKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvbmcgPSBjb2xfZG91YmxlKCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhbHQgPSBjb2xfZG91YmxlKCksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhY3Rpdml0eSA9IGNvbF9jaGFyYWN0ZXIoKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlkID0gY29sX2NoYXJhY3RlcigpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKQopCgpgYGAKCmBgYHtyfQpjYXByaW5vX2RhdGEKYGBgCgojIyMgTUFJTgojIyMjIEJhZyBvZiBmZWF0dXJlcyB1c2luZyBrbWVhbnMKYGBge3J9CmNsdXN0ZXJfZmVhdHVyZXM8LWMoJ2FjY1gnLCdhY2NZJywnYWNjWicsJ2d5cm9YJywnZ3lyb1knLCdneXJvWicpCgogIG1pbnV0ZV9zbGljZSA8LQogICAgYXMudmVjdG9yKHQoCiAgICAgIGNhcHJpbm9fZGF0YSAlPiUgZ3JvdXBfYnkoaWQpICU+JSAKICAgICAgICBzdW1tYXJpc2UobiA9IG4oKSkgJT4lIHNlbGVjdChpZCkKICAgICkpCgogIHRyYWluaWR4IDwtIHNhbXBsZSgxOmxlbmd0aChtaW51dGVfc2xpY2UpLCAKICAgICAgICAgICAgICAgICAgICAgKGxlbmd0aChtaW51dGVfc2xpY2UpICogNzApIC8gMTAwLjApCiAgZGF0YV90cmFpbiA8LSBjYXByaW5vX2RhdGEgJT4lIAogICAgZmlsdGVyKGlkICVpbiUgbWludXRlX3NsaWNlW3RyYWluaWR4XSkKICBkYXRhX3Rlc3QgPC0gY2Fwcmlub19kYXRhICU+JSAKICAgIGZpbHRlcihpZCAlaW4lIChtaW51dGVfc2xpY2VbLXRyYWluaWR4XSkpCiAgCiAgY2x1c3Rlcl9yZXN1bHRzIDwtCiAgICBrbWVhbnMoCiAgICAgIHggPSBkYXRhX3RyYWluICU+JSBzZWxlY3QoY2x1c3Rlcl9mZWF0dXJlcyksCiAgICAgIGNlbnRlcnMgPSAyNCwKICAgICAgbnN0YXJ0ID0gNQogICAgKQogIGRhdGFfdHJhaW5fY2x1c3RlciA8LQogICAgY2JpbmQoZGF0YV90cmFpbiwgY2x1c3RlciA9IGNsdXN0ZXJfcmVzdWx0cyRjbHVzdGVyKSAlPiUgCiAgICBncm91cF9ieShhY3Rpdml0eSwgY2x1c3RlcikKICAKICBjbHVzdGVyX3Jlc3VsdHNfdGVzdCA8LQogICAga21lYW5zKHggPSBkYXRhX3Rlc3QgJT4lIHNlbGVjdChjbHVzdGVyX2ZlYXR1cmVzKSwKICAgICAgICAgICBjZW50ZXJzID0gY2x1c3Rlcl9yZXN1bHRzJGNlbnRlcnMpCiAgZGF0YV90ZXN0X2NsdXN0ZXIgPC0KICAgIGNiaW5kKGRhdGFfdGVzdCwgY2x1c3RlciA9IGNsdXN0ZXJfcmVzdWx0c190ZXN0JGNsdXN0ZXIpICU+JSAKICAgIGdyb3VwX2J5KGFjdGl2aXR5LCBjbHVzdGVyKQogIAogIAogIGRhdGFfdHJhaW5fY2x1c3Rlcl9sYWJlbGVkIDwtIGRhdGFfdHJhaW5fY2x1c3RlciAgICU+JQogICAgZ3JvdXBfYnkoaWQsIGNsdXN0ZXIpICU+JSBzdW1tYXJpc2UobiA9IG4oKSkgJT4lCiAgICByZXNoYXBlMjo6ZGNhc3QoaWQgfiBjbHVzdGVyLCBmaWxsID0gMCkgJT4lCiAgICBpbm5lcl9qb2luKGRhdGFfdHJhaW5fY2x1c3RlciAlPiUgdW5ncm91cCgpICU+JQogICAgICAgICAgICAgICAgIHNlbGVjdChpZCwgYWN0aXZpdHkpICAlPiUKICAgICAgICAgICAgICAgICB1bmlxdWUoKSAgLAogICAgICAgICAgICAgICBieSA9ICJpZCIpICU+JSBmaWx0ZXIoIWlzLm5hKGFjdGl2aXR5KSkKICBkYXRhX3Rlc3RfY2x1c3Rlcl9sYWJlbGVkIDwtIGRhdGFfdGVzdF9jbHVzdGVyICAgJT4lIAogICAgZ3JvdXBfYnkoaWQsIGNsdXN0ZXIpICU+JSBzdW1tYXJpc2UobiA9IG4oKSkgJT4lCiAgICByZXNoYXBlMjo6ZGNhc3QoaWQgfiBjbHVzdGVyLCBmaWxsID0gMCkgJT4lCiAgICBpbm5lcl9qb2luKGRhdGFfdGVzdF9jbHVzdGVyICU+JSAgdW5ncm91cCgpICU+JQogICAgICAgICAgICAgICAgIHNlbGVjdChpZCwgYWN0aXZpdHkpICAlPiUKICAgICAgICAgICAgICAgICB1bmlxdWUoKSAsCiAgICAgICAgICAgICAgIGJ5ID0gImlkIikgICU+JSBmaWx0ZXIoIWlzLm5hKGFjdGl2aXR5KSkKICAKICBkYXRhX2V2YWxfY2x1c3Rlcl9sYWJlbGVkX2lkeCA8LSBzYW1wbGUoMTpucm93KGRhdGFfdHJhaW5fY2x1c3Rlcl9sYWJlbGVkKSwgCiAgICAgICAgICAgICAgICAgICAgIChsZW5ndGgoZGF0YV90cmFpbl9jbHVzdGVyX2xhYmVsZWQpICogNzApIC8gMTAwLjApCiAgZGF0YV9ldmFsX2NsdXN0ZXJfbGFiZWxlZCA8LSBkYXRhX3RyYWluX2NsdXN0ZXJfbGFiZWxlZFtkYXRhX2V2YWxfY2x1c3Rlcl9sYWJlbGVkX2lkeCxdCiAgZGF0YV90cmFpbl9jbHVzdGVyX2xhYmVsZWQgPC0gZGF0YV90cmFpbl9jbHVzdGVyX2xhYmVsZWRbLWRhdGFfZXZhbF9jbHVzdGVyX2xhYmVsZWRfaWR4LF0KCmBgYAoKIyMjIyBjcmVhdGUgcG9vbCBvYmplY3RzIGFzIHJlcXVpcmVkIGJ5IGNhdGJvb3N0IFIgcGFja2FnZQpgYGB7cn0KcG9vbF90cmFpbiA9IGNhdGJvb3N0LmxvYWRfcG9vbCggZGF0YV90cmFpbl9jbHVzdGVyX2xhYmVsZWQgJT4lIHNlbGVjdCgtaWQsLWFjdGl2aXR5KSwgbGFiZWwgPSBhcy5udW1lcmljKGFzLmZhY3RvcihkYXRhX3RyYWluX2NsdXN0ZXJfbGFiZWxlZCRhY3Rpdml0eSkpLTEpCnBvb2xfdGVzdCA9IGNhdGJvb3N0LmxvYWRfcG9vbCggZGF0YV90ZXN0X2NsdXN0ZXJfbGFiZWxlZCAlPiUgc2VsZWN0KC1pZCwtYWN0aXZpdHkpLCBsYWJlbCA9IGFzLm51bWVyaWMoYXMuZmFjdG9yKGRhdGFfdGVzdF9jbHVzdGVyX2xhYmVsZWQkYWN0aXZpdHkpKS0xKQpwb29sX3ZhbGlkID0gY2F0Ym9vc3QubG9hZF9wb29sKCBkYXRhX2V2YWxfY2x1c3Rlcl9sYWJlbGVkICU+JSBzZWxlY3QoLWlkLC1hY3Rpdml0eSksIGxhYmVsID0gYXMubnVtZXJpYyhhcy5mYWN0b3IoZGF0YV9ldmFsX2NsdXN0ZXJfbGFiZWxlZCRhY3Rpdml0eSkpLTEpCmBgYAoKIyMjIyBBbmFseXNpcyBvZiB0aGUgQ1YgcmVzdWx0cwpUaGUgaWRlYSBvZiB0aGVzZSBhbmFsaXN5cyB3YXMgdG8gY2hlY2sgb3ZlcmZpdHRpbmcuCgpgYGB7cn0KCmN2X3Jlc3VsdHMgPC0gY2F0Ym9vc3QuY3YocG9vbF90cmFpbixmb2xkX2NvdW50ID0gMyxlYXJseV9zdG9wcGluZ19yb3VuZHM9NDApCmN2X3Jlc3VsdHMgJT4lIHNlbGVjdCh0ZXN0LlJNU0UubWVhbiwgdHJhaW4uUk1TRS5tZWFuKSAlPiUgdGliYmxlOjphZGRfY29sdW1uKGlkPXNlcSgxOm5yb3coY3ZfcmVzdWx0cykpKSAlPiUgcmVzaGFwZTI6Om1lbHQoaWQudmFyPSJpZCIpICU+JQpnZ3Bsb3QoYWVzKHg9aWQseT12YWx1ZSkpKwogICBnZW9tX2xpbmUoYWVzKGNvbG9yPXZhcmlhYmxlLGxpbmV0eXBlPXZhcmlhYmxlKSkrCiAgIGdnZGFyazo6ZGFya190aGVtZV9ncmF5KCkKYGBgCgojIyMjIFRyYWluaW5nIHRoZSBtb2RlbApBIGxhcmdlIGl0ZXJhdGlvbiBidXQgd2UgdXNlIGVhcmx5IHN0b3BpbmcKCmBgYHtyfQpmaXRfcGFyYW1zIDwtIGxpc3QoaXRlcmF0aW9ucyA9IDEwMDAsCiAgICAgICAgICAgICAgICAgICBsb3NzX2Z1bmN0aW9uID0gJ011bHRpQ2xhc3MnLAogICAgICAgICAgICAgICAgICAgI2JvcmRlcl9jb3VudCA9IDMyLAogICAgICAgICAgICAgICAgICAgZGVwdGggPSAzLAogICAgICAgICAgICAgICAgICAgI2xlYXJuaW5nX3JhdGUgPSAwLjAzLAogICAgICAgICAgICAgICAgICAgI2wyX2xlYWZfcmVnID0gMy41LAogICAgICAgICAgICAgICAgICAgdGFza190eXBlID0gJ0NQVScsCiAgICAgICAgICAgICAgICAgICB2ZXJib3NlID0gMTAsCiAgICAgICAgICAgICAgICAgICBlYXJseV9zdG9wcGluZ19yb3VuZHM9NDAKICAgICAgICAgICAgICAgICAgICkKbW9kZWwgPC0gY2F0Ym9vc3QudHJhaW4ocG9vbF90cmFpbiwgcG9vbF92YWxpZCwgcGFyYW1zID0gZml0X3BhcmFtcykKYGBgCgojIyMjIENoZWNrIHRoZSByZWxhdGlvbiBiZXR3ZWVuIHRoZSBsYWJlbHMgdXNlZCBpbiBSRiBhbmQgdGhlIGxhYmVscyB1c2VkIGluIGNhdGJvb3N0CgpgYGB7cn0KYXMuZmFjdG9yKGRhdGFfdGVzdF9jbHVzdGVyX2xhYmVsZWQkYWN0aXZpdHkpCmFzLmZhY3Rvcihhcy5udW1lcmljKGFzLmZhY3RvcihkYXRhX3Rlc3RfY2x1c3Rlcl9sYWJlbGVkJGFjdGl2aXR5KSktMSkKYGBgCiAgCiMjIyMgQ29uZnVzaW9uIE1hdHJpeCBhbmQgbWFjcm8gYW5kIG1pY3JvIG1ldHJpY3MKYGBge3J9CiAgZGF0YV90ZXN0X3ByZWRpY3Rpb24gPC1jYXRib29zdC5wcmVkaWN0KG1vZGVsLCBwb29sX3Rlc3QscHJlZGljdGlvbl90eXBlPSdDbGFzcycpCiAgY20gPC0KICAgIGNhcmV0Ojpjb25mdXNpb25NYXRyaXgoYXMuZmFjdG9yKGRhdGFfdGVzdF9wcmVkaWN0aW9uKSxhcy5mYWN0b3IoYXMubnVtZXJpYyhhcy5mYWN0b3IoZGF0YV90ZXN0X2NsdXN0ZXJfbGFiZWxlZCRhY3Rpdml0eSkpLTEpLAogICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlID0gJ2V2ZXJ5dGhpbmcnKQoKY20KCmBgYAoKYGBge3J9CiAgbWFjcm9fbWV0cmljcyA8LSAoYXBwbHkoY20kYnlDbGFzcywgMiwgc3VtKSAvIDQpIFtjKDUsIDYsIDcpXQogIHMgPC0KICAgIHRhYmxlKGRhdGFfdGVzdF9wcmVkaWN0aW9uLAogICAgICAgICAgYXMuZmFjdG9yKGRhdGFfdGVzdF9jbHVzdGVyX2xhYmVsZWQkYWN0aXZpdHkpKQogIAogIFByZWNpc2lvbl9taWNybyA8LSAgKHN1bShkaWFnKHMpKSAvIHN1bShhcHBseShzLCAxLCBzdW0pKSkKICBSZWNhbGxfbWljcm8gPC0gKHN1bShkaWFnKHMpKSAvIHN1bShhcHBseShzLCAyLCBzdW0pKSkKICAKICBtaWNyb19tZXRyaWNzIDwtIGMoKHN1bShkaWFnKHMpKSAvIHN1bShhcHBseShzLCAxLCBzdW0pKSksCiAgICAgICAgICAgICAgICAgICAgIChzdW0oZGlhZyhzKSkgLyBzdW0oYXBwbHkocywgMiwgc3VtKSkpLAogICAgICAgICAgICAgICAgICAgICAyICogKChQcmVjaXNpb25fbWljcm8gKiBSZWNhbGxfbWljcm8pIC8gKFByZWNpc2lvbl9taWNybyArIFJlY2FsbF9taWNybykKICAgICAgICAgICAgICAgICAgICAgKSkKICAKICBtbSA8LSBjYmluZChjKG1hY3JvX21ldHJpY3MpLCBjKG1pY3JvX21ldHJpY3MpKQptbQpgYGAKCiMjIENvbmZ1c2lvbiAgTWF0cml4IChwbG90KQpgYGB7cn0KbGVuX2RhdGFfdGVzdF9wcmVkaWN0aW9uID0gbGVuZ3RoKGRhdGFfdGVzdF9wcmVkaWN0aW9uKQpyZXNoYXBlMjo6bWVsdCh0YWJsZShkYXRhX3Rlc3RfcHJlZGljdGlvbixhcy5mYWN0b3IoZGF0YV90ZXN0X2NsdXN0ZXJfbGFiZWxlZCRhY3Rpdml0eSkpKSAlPiUgbXV0YXRlKHZhbHVlPSh2YWx1ZSkpICU+JQogIGdncGxvdChhZXMoeD1kYXRhX3Rlc3RfcHJlZGljdGlvbix5PVZhcjIpKSsKICBnZW9tX3RpbGUoYWVzKGZpbGw9dmFsdWUpLCBjb2xvdXIgPSAid2hpdGUiKSArIAogICBnZW9tX3RleHQoYWVzKGxhYmVsID0gc3ByaW50ZigiJTEuMmYiLCB2YWx1ZSkpLCB2anVzdCA9IDEpKwogIHNjYWxlX2ZpbGxfZ3JhZGllbnQobG93ID0gIndoaXRlIiwgaGlnaCA9ICJyZWQiKSsKICB4bGFiKCIgUHJlZGljdGVkIEFjdGl2aXR5ICIpK3lsYWIoIiBBY3R1YWwgQWN0aXZpdHkiKSsKICB0aGVtZV9idygpKyB0aGVtZShsZWdlbmQucG9zaXRpb24gPSAibm9uZSIpCmBgYApgYGB7cn0Kc2Vzc2lvbkluZm8oKQpgYGAKCg==