library(tidyverse)
## ── Attaching packages ────────────────────────────────────────────────────── tidyverse 1.2.1 ──
## ✔ ggplot2 3.2.1 ✔ purrr 0.3.2
## ✔ tibble 2.1.3 ✔ dplyr 0.8.3
## ✔ tidyr 1.0.0 ✔ stringr 1.4.0
## ✔ readr 1.3.1 ✔ forcats 0.4.0
## ── Conflicts ───────────────────────────────────────────────────────── tidyverse_conflicts() ──
## ✖ dplyr::filter() masks stats::filter()
## ✖ dplyr::lag() masks stats::lag()
library(quanteda)
## Package version: 1.5.1
## Parallel computing: 2 of 8 threads used.
## See https://quanteda.io for tutorials and examples.
##
## Attaching package: 'quanteda'
## The following object is masked from 'package:utils':
##
## View
a <- read_csv("all-new-data.csv") %>%
mutate(Why = text) %>%
mutate(Why = str_c(Why, " ", lesson, " ", gen_or_spec)) %>%
mutate(row_num = row_number(),
code = round(code),
#code = ifelse(code == 5, 4, code),
Why = ifelse(is.na(Why), "blank", Why))
## Parsed with column specification:
## cols(
## ID = col_double(),
## lesson = col_character(),
## gen_or_spec = col_character(),
## text = col_character(),
## code = col_double(),
## ID_in_7C = col_logical()
## )
resample <- function(gd, code, row, classifier = "nb", verbose = F) {
gd <- gd %>%
select(text = Why, everything())
gds <- gd[row, ]
gd <- gd[-row, ]
gd <- corpus(gd)
gds <- corpus(gds)
dfm_training <- dfm(gd, stem = TRUE, ngrams = 1)
dfm_test <- dfm(gds, stem = TRUE, ngrams = 1)
if (classifier == "svm") {
dfm_training <- dfm_training
m1 <- quanteda.classifiers::textmodel_svm(dfm_training, docvars(dfm_training, code))
} else if (classifier == "nn") {
dfm_training <- dfm_training
m1 <- quanteda.classifiers::textmodel_nnseq(dfm_training, docvars(dfm_training, code),
epochs = 6, verbose = verbose)
} else if (classifier == "nb") {
m1 <- textmodel_nb(dfm_training, docvars(dfm_training, code))
}
dfmat_matched <- dfm_match(dfm_test, features = featnames(dfm_training))
actual_class <- docvars(dfmat_matched, code)
predicted_class <- predict(m1, newdata = dfmat_matched)
if (verbose == verbose) {
print(str_c("### Completed bootstrap sample: ", row, " ###"))
}
data.frame(actual_class = actual_class, predicted_class = predicted_class)
}
get_performance <- function(do) {
# https://www.dataschool.io/simple-guide-to-confusion-matrix-terminology/
tab_class <- table(do$actual_class, do$predicted_class)
c <- caret::confusionMatrix(tab_class, mode = "everything")
return(c)
}
run_model <- function(gd, code, .f = resample, classifier) {
do <- map_df(.x = seq(nrow(gd)),
gd = gd,
code = code,
.f = .f,
classifier = classifier)
do
get_performance(do)
}
nsvm3 <- run_model(gd = a, code = "code", classifier = "svm")
## [1] "### Completed bootstrap sample: 1 ###"
## [1] "### Completed bootstrap sample: 2 ###"
## [1] "### Completed bootstrap sample: 3 ###"
## [1] "### Completed bootstrap sample: 4 ###"
## [1] "### Completed bootstrap sample: 5 ###"
## [1] "### Completed bootstrap sample: 6 ###"
## [1] "### Completed bootstrap sample: 7 ###"
## [1] "### Completed bootstrap sample: 8 ###"
## [1] "### Completed bootstrap sample: 9 ###"
## [1] "### Completed bootstrap sample: 10 ###"
## [1] "### Completed bootstrap sample: 11 ###"
## [1] "### Completed bootstrap sample: 12 ###"
## [1] "### Completed bootstrap sample: 13 ###"
## [1] "### Completed bootstrap sample: 14 ###"
## [1] "### Completed bootstrap sample: 15 ###"
## [1] "### Completed bootstrap sample: 16 ###"
## [1] "### Completed bootstrap sample: 17 ###"
## [1] "### Completed bootstrap sample: 18 ###"
## [1] "### Completed bootstrap sample: 19 ###"
## [1] "### Completed bootstrap sample: 20 ###"
## [1] "### Completed bootstrap sample: 21 ###"
## [1] "### Completed bootstrap sample: 22 ###"
## [1] "### Completed bootstrap sample: 23 ###"
## [1] "### Completed bootstrap sample: 24 ###"
## [1] "### Completed bootstrap sample: 25 ###"
## [1] "### Completed bootstrap sample: 26 ###"
## [1] "### Completed bootstrap sample: 27 ###"
## [1] "### Completed bootstrap sample: 28 ###"
## [1] "### Completed bootstrap sample: 29 ###"
## [1] "### Completed bootstrap sample: 30 ###"
## [1] "### Completed bootstrap sample: 31 ###"
## [1] "### Completed bootstrap sample: 32 ###"
## [1] "### Completed bootstrap sample: 33 ###"
## [1] "### Completed bootstrap sample: 34 ###"
## [1] "### Completed bootstrap sample: 35 ###"
## [1] "### Completed bootstrap sample: 36 ###"
## [1] "### Completed bootstrap sample: 37 ###"
## [1] "### Completed bootstrap sample: 38 ###"
## [1] "### Completed bootstrap sample: 39 ###"
## [1] "### Completed bootstrap sample: 40 ###"
## [1] "### Completed bootstrap sample: 41 ###"
## [1] "### Completed bootstrap sample: 42 ###"
## [1] "### Completed bootstrap sample: 43 ###"
## [1] "### Completed bootstrap sample: 44 ###"
## [1] "### Completed bootstrap sample: 45 ###"
## [1] "### Completed bootstrap sample: 46 ###"
## [1] "### Completed bootstrap sample: 47 ###"
## [1] "### Completed bootstrap sample: 48 ###"
## [1] "### Completed bootstrap sample: 49 ###"
## [1] "### Completed bootstrap sample: 50 ###"
## [1] "### Completed bootstrap sample: 51 ###"
## [1] "### Completed bootstrap sample: 52 ###"
## [1] "### Completed bootstrap sample: 53 ###"
## [1] "### Completed bootstrap sample: 54 ###"
## [1] "### Completed bootstrap sample: 55 ###"
## [1] "### Completed bootstrap sample: 56 ###"
## [1] "### Completed bootstrap sample: 57 ###"
## [1] "### Completed bootstrap sample: 58 ###"
## [1] "### Completed bootstrap sample: 59 ###"
## [1] "### Completed bootstrap sample: 60 ###"
## [1] "### Completed bootstrap sample: 61 ###"
## [1] "### Completed bootstrap sample: 62 ###"
## [1] "### Completed bootstrap sample: 63 ###"
## [1] "### Completed bootstrap sample: 64 ###"
## [1] "### Completed bootstrap sample: 65 ###"
## [1] "### Completed bootstrap sample: 66 ###"
## [1] "### Completed bootstrap sample: 67 ###"
## [1] "### Completed bootstrap sample: 68 ###"
## [1] "### Completed bootstrap sample: 69 ###"
## [1] "### Completed bootstrap sample: 70 ###"
## [1] "### Completed bootstrap sample: 71 ###"
## [1] "### Completed bootstrap sample: 72 ###"
## [1] "### Completed bootstrap sample: 73 ###"
## [1] "### Completed bootstrap sample: 74 ###"
## [1] "### Completed bootstrap sample: 75 ###"
## [1] "### Completed bootstrap sample: 76 ###"
## [1] "### Completed bootstrap sample: 77 ###"
## [1] "### Completed bootstrap sample: 78 ###"
## [1] "### Completed bootstrap sample: 79 ###"
## [1] "### Completed bootstrap sample: 80 ###"
## [1] "### Completed bootstrap sample: 81 ###"
## [1] "### Completed bootstrap sample: 82 ###"
## [1] "### Completed bootstrap sample: 83 ###"
## [1] "### Completed bootstrap sample: 84 ###"
## [1] "### Completed bootstrap sample: 85 ###"
## [1] "### Completed bootstrap sample: 86 ###"
## [1] "### Completed bootstrap sample: 87 ###"
## [1] "### Completed bootstrap sample: 88 ###"
## [1] "### Completed bootstrap sample: 89 ###"
## [1] "### Completed bootstrap sample: 90 ###"
## [1] "### Completed bootstrap sample: 91 ###"
## [1] "### Completed bootstrap sample: 92 ###"
## [1] "### Completed bootstrap sample: 93 ###"
## [1] "### Completed bootstrap sample: 94 ###"
## [1] "### Completed bootstrap sample: 95 ###"
## [1] "### Completed bootstrap sample: 96 ###"
## [1] "### Completed bootstrap sample: 97 ###"
## [1] "### Completed bootstrap sample: 98 ###"
## [1] "### Completed bootstrap sample: 99 ###"
## [1] "### Completed bootstrap sample: 100 ###"
## [1] "### Completed bootstrap sample: 101 ###"
## [1] "### Completed bootstrap sample: 102 ###"
## [1] "### Completed bootstrap sample: 103 ###"
## [1] "### Completed bootstrap sample: 104 ###"
## [1] "### Completed bootstrap sample: 105 ###"
## [1] "### Completed bootstrap sample: 106 ###"
## [1] "### Completed bootstrap sample: 107 ###"
## [1] "### Completed bootstrap sample: 108 ###"
## [1] "### Completed bootstrap sample: 109 ###"
## [1] "### Completed bootstrap sample: 110 ###"
## [1] "### Completed bootstrap sample: 111 ###"
## [1] "### Completed bootstrap sample: 112 ###"
## [1] "### Completed bootstrap sample: 113 ###"
## [1] "### Completed bootstrap sample: 114 ###"
## [1] "### Completed bootstrap sample: 115 ###"
## [1] "### Completed bootstrap sample: 116 ###"
## [1] "### Completed bootstrap sample: 117 ###"
## [1] "### Completed bootstrap sample: 118 ###"
## [1] "### Completed bootstrap sample: 119 ###"
## [1] "### Completed bootstrap sample: 120 ###"
## [1] "### Completed bootstrap sample: 121 ###"
## [1] "### Completed bootstrap sample: 122 ###"
## [1] "### Completed bootstrap sample: 123 ###"
## [1] "### Completed bootstrap sample: 124 ###"
## [1] "### Completed bootstrap sample: 125 ###"
## [1] "### Completed bootstrap sample: 126 ###"
## [1] "### Completed bootstrap sample: 127 ###"
## [1] "### Completed bootstrap sample: 128 ###"
## [1] "### Completed bootstrap sample: 129 ###"
## [1] "### Completed bootstrap sample: 130 ###"
## [1] "### Completed bootstrap sample: 131 ###"
## [1] "### Completed bootstrap sample: 132 ###"
## [1] "### Completed bootstrap sample: 133 ###"
## [1] "### Completed bootstrap sample: 134 ###"
## [1] "### Completed bootstrap sample: 135 ###"
## [1] "### Completed bootstrap sample: 136 ###"
## [1] "### Completed bootstrap sample: 137 ###"
## [1] "### Completed bootstrap sample: 138 ###"
## [1] "### Completed bootstrap sample: 139 ###"
## [1] "### Completed bootstrap sample: 140 ###"
## [1] "### Completed bootstrap sample: 141 ###"
## [1] "### Completed bootstrap sample: 142 ###"
## [1] "### Completed bootstrap sample: 143 ###"
## [1] "### Completed bootstrap sample: 144 ###"
## [1] "### Completed bootstrap sample: 145 ###"
## [1] "### Completed bootstrap sample: 146 ###"
## [1] "### Completed bootstrap sample: 147 ###"
## [1] "### Completed bootstrap sample: 148 ###"
## [1] "### Completed bootstrap sample: 149 ###"
## [1] "### Completed bootstrap sample: 150 ###"
## [1] "### Completed bootstrap sample: 151 ###"
## [1] "### Completed bootstrap sample: 152 ###"
## [1] "### Completed bootstrap sample: 153 ###"
## [1] "### Completed bootstrap sample: 154 ###"
## [1] "### Completed bootstrap sample: 155 ###"
## [1] "### Completed bootstrap sample: 156 ###"
## [1] "### Completed bootstrap sample: 157 ###"
## [1] "### Completed bootstrap sample: 158 ###"
## [1] "### Completed bootstrap sample: 159 ###"
## [1] "### Completed bootstrap sample: 160 ###"
## [1] "### Completed bootstrap sample: 161 ###"
## [1] "### Completed bootstrap sample: 162 ###"
## [1] "### Completed bootstrap sample: 163 ###"
## [1] "### Completed bootstrap sample: 164 ###"
## [1] "### Completed bootstrap sample: 165 ###"
## [1] "### Completed bootstrap sample: 166 ###"
## [1] "### Completed bootstrap sample: 167 ###"
## [1] "### Completed bootstrap sample: 168 ###"
## [1] "### Completed bootstrap sample: 169 ###"
## [1] "### Completed bootstrap sample: 170 ###"
## [1] "### Completed bootstrap sample: 171 ###"
## [1] "### Completed bootstrap sample: 172 ###"
## [1] "### Completed bootstrap sample: 173 ###"
## [1] "### Completed bootstrap sample: 174 ###"
## [1] "### Completed bootstrap sample: 175 ###"
## [1] "### Completed bootstrap sample: 176 ###"
## [1] "### Completed bootstrap sample: 177 ###"
## [1] "### Completed bootstrap sample: 178 ###"
## [1] "### Completed bootstrap sample: 179 ###"
## [1] "### Completed bootstrap sample: 180 ###"
## [1] "### Completed bootstrap sample: 181 ###"
## [1] "### Completed bootstrap sample: 182 ###"
## [1] "### Completed bootstrap sample: 183 ###"
## [1] "### Completed bootstrap sample: 184 ###"
## [1] "### Completed bootstrap sample: 185 ###"
## [1] "### Completed bootstrap sample: 186 ###"
## [1] "### Completed bootstrap sample: 187 ###"
## [1] "### Completed bootstrap sample: 188 ###"
## [1] "### Completed bootstrap sample: 189 ###"
## [1] "### Completed bootstrap sample: 190 ###"
## [1] "### Completed bootstrap sample: 191 ###"
## [1] "### Completed bootstrap sample: 192 ###"
## [1] "### Completed bootstrap sample: 193 ###"
## [1] "### Completed bootstrap sample: 194 ###"
## [1] "### Completed bootstrap sample: 195 ###"
## [1] "### Completed bootstrap sample: 196 ###"
## [1] "### Completed bootstrap sample: 197 ###"
## [1] "### Completed bootstrap sample: 198 ###"
## [1] "### Completed bootstrap sample: 199 ###"
## [1] "### Completed bootstrap sample: 200 ###"
## [1] "### Completed bootstrap sample: 201 ###"
## [1] "### Completed bootstrap sample: 202 ###"
## [1] "### Completed bootstrap sample: 203 ###"
## [1] "### Completed bootstrap sample: 204 ###"
## [1] "### Completed bootstrap sample: 205 ###"
## [1] "### Completed bootstrap sample: 206 ###"
## [1] "### Completed bootstrap sample: 207 ###"
## [1] "### Completed bootstrap sample: 208 ###"
## [1] "### Completed bootstrap sample: 209 ###"
## [1] "### Completed bootstrap sample: 210 ###"
## [1] "### Completed bootstrap sample: 211 ###"
## [1] "### Completed bootstrap sample: 212 ###"
## [1] "### Completed bootstrap sample: 213 ###"
## [1] "### Completed bootstrap sample: 214 ###"
## [1] "### Completed bootstrap sample: 215 ###"
## [1] "### Completed bootstrap sample: 216 ###"
## [1] "### Completed bootstrap sample: 217 ###"
## [1] "### Completed bootstrap sample: 218 ###"
## [1] "### Completed bootstrap sample: 219 ###"
## [1] "### Completed bootstrap sample: 220 ###"
## [1] "### Completed bootstrap sample: 221 ###"
## [1] "### Completed bootstrap sample: 222 ###"
## [1] "### Completed bootstrap sample: 223 ###"
## [1] "### Completed bootstrap sample: 224 ###"
## [1] "### Completed bootstrap sample: 225 ###"
## [1] "### Completed bootstrap sample: 226 ###"
## [1] "### Completed bootstrap sample: 227 ###"
## [1] "### Completed bootstrap sample: 228 ###"
## [1] "### Completed bootstrap sample: 229 ###"
## [1] "### Completed bootstrap sample: 230 ###"
## [1] "### Completed bootstrap sample: 231 ###"
## [1] "### Completed bootstrap sample: 232 ###"
## [1] "### Completed bootstrap sample: 233 ###"
## [1] "### Completed bootstrap sample: 234 ###"
## [1] "### Completed bootstrap sample: 235 ###"
## [1] "### Completed bootstrap sample: 236 ###"
## [1] "### Completed bootstrap sample: 237 ###"
## [1] "### Completed bootstrap sample: 238 ###"
## [1] "### Completed bootstrap sample: 239 ###"
## [1] "### Completed bootstrap sample: 240 ###"
## [1] "### Completed bootstrap sample: 241 ###"
## [1] "### Completed bootstrap sample: 242 ###"
## [1] "### Completed bootstrap sample: 243 ###"
## [1] "### Completed bootstrap sample: 244 ###"
## [1] "### Completed bootstrap sample: 245 ###"
## [1] "### Completed bootstrap sample: 246 ###"
## [1] "### Completed bootstrap sample: 247 ###"
## [1] "### Completed bootstrap sample: 248 ###"
## [1] "### Completed bootstrap sample: 249 ###"
## [1] "### Completed bootstrap sample: 250 ###"
## [1] "### Completed bootstrap sample: 251 ###"
## [1] "### Completed bootstrap sample: 252 ###"
## [1] "### Completed bootstrap sample: 253 ###"
## [1] "### Completed bootstrap sample: 254 ###"
## [1] "### Completed bootstrap sample: 255 ###"
## [1] "### Completed bootstrap sample: 256 ###"
## [1] "### Completed bootstrap sample: 257 ###"
## [1] "### Completed bootstrap sample: 258 ###"
## [1] "### Completed bootstrap sample: 259 ###"
## [1] "### Completed bootstrap sample: 260 ###"
## [1] "### Completed bootstrap sample: 261 ###"
## [1] "### Completed bootstrap sample: 262 ###"
## [1] "### Completed bootstrap sample: 263 ###"
## [1] "### Completed bootstrap sample: 264 ###"
## [1] "### Completed bootstrap sample: 265 ###"
## [1] "### Completed bootstrap sample: 266 ###"
## [1] "### Completed bootstrap sample: 267 ###"
## [1] "### Completed bootstrap sample: 268 ###"
## [1] "### Completed bootstrap sample: 269 ###"
## [1] "### Completed bootstrap sample: 270 ###"
## [1] "### Completed bootstrap sample: 271 ###"
## [1] "### Completed bootstrap sample: 272 ###"
## [1] "### Completed bootstrap sample: 273 ###"
## [1] "### Completed bootstrap sample: 274 ###"
## [1] "### Completed bootstrap sample: 275 ###"
## [1] "### Completed bootstrap sample: 276 ###"
## [1] "### Completed bootstrap sample: 277 ###"
## [1] "### Completed bootstrap sample: 278 ###"
## [1] "### Completed bootstrap sample: 279 ###"
## [1] "### Completed bootstrap sample: 280 ###"
## [1] "### Completed bootstrap sample: 281 ###"
## [1] "### Completed bootstrap sample: 282 ###"
## [1] "### Completed bootstrap sample: 283 ###"
## [1] "### Completed bootstrap sample: 284 ###"
## [1] "### Completed bootstrap sample: 285 ###"
## [1] "### Completed bootstrap sample: 286 ###"
## [1] "### Completed bootstrap sample: 287 ###"
## [1] "### Completed bootstrap sample: 288 ###"
## [1] "### Completed bootstrap sample: 289 ###"
## [1] "### Completed bootstrap sample: 290 ###"
## [1] "### Completed bootstrap sample: 291 ###"
## [1] "### Completed bootstrap sample: 292 ###"
## [1] "### Completed bootstrap sample: 293 ###"
## [1] "### Completed bootstrap sample: 294 ###"
## [1] "### Completed bootstrap sample: 295 ###"
## [1] "### Completed bootstrap sample: 296 ###"
## [1] "### Completed bootstrap sample: 297 ###"
## [1] "### Completed bootstrap sample: 298 ###"
## [1] "### Completed bootstrap sample: 299 ###"
## [1] "### Completed bootstrap sample: 300 ###"
## [1] "### Completed bootstrap sample: 301 ###"
## [1] "### Completed bootstrap sample: 302 ###"
## [1] "### Completed bootstrap sample: 303 ###"
## [1] "### Completed bootstrap sample: 304 ###"
## [1] "### Completed bootstrap sample: 305 ###"
## [1] "### Completed bootstrap sample: 306 ###"
## [1] "### Completed bootstrap sample: 307 ###"
## [1] "### Completed bootstrap sample: 308 ###"
## [1] "### Completed bootstrap sample: 309 ###"
## [1] "### Completed bootstrap sample: 310 ###"
## [1] "### Completed bootstrap sample: 311 ###"
## [1] "### Completed bootstrap sample: 312 ###"
## [1] "### Completed bootstrap sample: 313 ###"
## [1] "### Completed bootstrap sample: 314 ###"
## [1] "### Completed bootstrap sample: 315 ###"
## [1] "### Completed bootstrap sample: 316 ###"
## [1] "### Completed bootstrap sample: 317 ###"
## [1] "### Completed bootstrap sample: 318 ###"
## [1] "### Completed bootstrap sample: 319 ###"
## [1] "### Completed bootstrap sample: 320 ###"
## [1] "### Completed bootstrap sample: 321 ###"
## [1] "### Completed bootstrap sample: 322 ###"
## [1] "### Completed bootstrap sample: 323 ###"
## [1] "### Completed bootstrap sample: 324 ###"
## [1] "### Completed bootstrap sample: 325 ###"
## [1] "### Completed bootstrap sample: 326 ###"
## [1] "### Completed bootstrap sample: 327 ###"
## [1] "### Completed bootstrap sample: 328 ###"
## [1] "### Completed bootstrap sample: 329 ###"
## [1] "### Completed bootstrap sample: 330 ###"
## [1] "### Completed bootstrap sample: 331 ###"
## [1] "### Completed bootstrap sample: 332 ###"
## [1] "### Completed bootstrap sample: 333 ###"
## [1] "### Completed bootstrap sample: 334 ###"
## [1] "### Completed bootstrap sample: 335 ###"
## [1] "### Completed bootstrap sample: 336 ###"
## [1] "### Completed bootstrap sample: 337 ###"
## [1] "### Completed bootstrap sample: 338 ###"
## [1] "### Completed bootstrap sample: 339 ###"
## [1] "### Completed bootstrap sample: 340 ###"
## [1] "### Completed bootstrap sample: 341 ###"
## [1] "### Completed bootstrap sample: 342 ###"
## [1] "### Completed bootstrap sample: 343 ###"
## [1] "### Completed bootstrap sample: 344 ###"
## [1] "### Completed bootstrap sample: 345 ###"
## [1] "### Completed bootstrap sample: 346 ###"
## [1] "### Completed bootstrap sample: 347 ###"
## [1] "### Completed bootstrap sample: 348 ###"
## [1] "### Completed bootstrap sample: 349 ###"
## [1] "### Completed bootstrap sample: 350 ###"
## [1] "### Completed bootstrap sample: 351 ###"
## [1] "### Completed bootstrap sample: 352 ###"
## [1] "### Completed bootstrap sample: 353 ###"
## [1] "### Completed bootstrap sample: 354 ###"
## [1] "### Completed bootstrap sample: 355 ###"
## [1] "### Completed bootstrap sample: 356 ###"
## [1] "### Completed bootstrap sample: 357 ###"
## [1] "### Completed bootstrap sample: 358 ###"
## [1] "### Completed bootstrap sample: 359 ###"
## [1] "### Completed bootstrap sample: 360 ###"
## [1] "### Completed bootstrap sample: 361 ###"
## [1] "### Completed bootstrap sample: 362 ###"
## [1] "### Completed bootstrap sample: 363 ###"
## [1] "### Completed bootstrap sample: 364 ###"
## [1] "### Completed bootstrap sample: 365 ###"
## [1] "### Completed bootstrap sample: 366 ###"
## [1] "### Completed bootstrap sample: 367 ###"
## [1] "### Completed bootstrap sample: 368 ###"
## [1] "### Completed bootstrap sample: 369 ###"
## [1] "### Completed bootstrap sample: 370 ###"
## [1] "### Completed bootstrap sample: 371 ###"
## [1] "### Completed bootstrap sample: 372 ###"
## [1] "### Completed bootstrap sample: 373 ###"
## [1] "### Completed bootstrap sample: 374 ###"
## [1] "### Completed bootstrap sample: 375 ###"
## [1] "### Completed bootstrap sample: 376 ###"
## [1] "### Completed bootstrap sample: 377 ###"
## [1] "### Completed bootstrap sample: 378 ###"
## [1] "### Completed bootstrap sample: 379 ###"
## [1] "### Completed bootstrap sample: 380 ###"
## [1] "### Completed bootstrap sample: 381 ###"
## [1] "### Completed bootstrap sample: 382 ###"
## [1] "### Completed bootstrap sample: 383 ###"
## [1] "### Completed bootstrap sample: 384 ###"
## [1] "### Completed bootstrap sample: 385 ###"
## [1] "### Completed bootstrap sample: 386 ###"
## [1] "### Completed bootstrap sample: 387 ###"
## [1] "### Completed bootstrap sample: 388 ###"
## [1] "### Completed bootstrap sample: 389 ###"
## [1] "### Completed bootstrap sample: 390 ###"
## [1] "### Completed bootstrap sample: 391 ###"
## [1] "### Completed bootstrap sample: 392 ###"
## [1] "### Completed bootstrap sample: 393 ###"
## [1] "### Completed bootstrap sample: 394 ###"
## [1] "### Completed bootstrap sample: 395 ###"
## [1] "### Completed bootstrap sample: 396 ###"
## [1] "### Completed bootstrap sample: 397 ###"
## [1] "### Completed bootstrap sample: 398 ###"
## [1] "### Completed bootstrap sample: 399 ###"
## [1] "### Completed bootstrap sample: 400 ###"
## [1] "### Completed bootstrap sample: 401 ###"
## [1] "### Completed bootstrap sample: 402 ###"
## [1] "### Completed bootstrap sample: 403 ###"
## [1] "### Completed bootstrap sample: 404 ###"
## [1] "### Completed bootstrap sample: 405 ###"
## [1] "### Completed bootstrap sample: 406 ###"
## [1] "### Completed bootstrap sample: 407 ###"
## [1] "### Completed bootstrap sample: 408 ###"
## [1] "### Completed bootstrap sample: 409 ###"
## [1] "### Completed bootstrap sample: 410 ###"
## [1] "### Completed bootstrap sample: 411 ###"
## [1] "### Completed bootstrap sample: 412 ###"
## [1] "### Completed bootstrap sample: 413 ###"
## [1] "### Completed bootstrap sample: 414 ###"
## [1] "### Completed bootstrap sample: 415 ###"
## [1] "### Completed bootstrap sample: 416 ###"
## [1] "### Completed bootstrap sample: 417 ###"
## [1] "### Completed bootstrap sample: 418 ###"
## [1] "### Completed bootstrap sample: 419 ###"
## [1] "### Completed bootstrap sample: 420 ###"
## [1] "### Completed bootstrap sample: 421 ###"
## [1] "### Completed bootstrap sample: 422 ###"
## [1] "### Completed bootstrap sample: 423 ###"
## [1] "### Completed bootstrap sample: 424 ###"
## [1] "### Completed bootstrap sample: 425 ###"
## [1] "### Completed bootstrap sample: 426 ###"
## [1] "### Completed bootstrap sample: 427 ###"
## [1] "### Completed bootstrap sample: 428 ###"
## [1] "### Completed bootstrap sample: 429 ###"
## [1] "### Completed bootstrap sample: 430 ###"
## [1] "### Completed bootstrap sample: 431 ###"
## [1] "### Completed bootstrap sample: 432 ###"
## [1] "### Completed bootstrap sample: 433 ###"
## [1] "### Completed bootstrap sample: 434 ###"
## [1] "### Completed bootstrap sample: 435 ###"
## [1] "### Completed bootstrap sample: 436 ###"
## [1] "### Completed bootstrap sample: 437 ###"
## [1] "### Completed bootstrap sample: 438 ###"
## [1] "### Completed bootstrap sample: 439 ###"
## [1] "### Completed bootstrap sample: 440 ###"
## [1] "### Completed bootstrap sample: 441 ###"
## [1] "### Completed bootstrap sample: 442 ###"
## [1] "### Completed bootstrap sample: 443 ###"
## [1] "### Completed bootstrap sample: 444 ###"
## [1] "### Completed bootstrap sample: 445 ###"
## [1] "### Completed bootstrap sample: 446 ###"
## [1] "### Completed bootstrap sample: 447 ###"
## [1] "### Completed bootstrap sample: 448 ###"
## [1] "### Completed bootstrap sample: 449 ###"
## [1] "### Completed bootstrap sample: 450 ###"
## [1] "### Completed bootstrap sample: 451 ###"
## [1] "### Completed bootstrap sample: 452 ###"
## [1] "### Completed bootstrap sample: 453 ###"
## [1] "### Completed bootstrap sample: 454 ###"
## [1] "### Completed bootstrap sample: 455 ###"
## [1] "### Completed bootstrap sample: 456 ###"
## [1] "### Completed bootstrap sample: 457 ###"
## [1] "### Completed bootstrap sample: 458 ###"
## [1] "### Completed bootstrap sample: 459 ###"
## [1] "### Completed bootstrap sample: 460 ###"
## [1] "### Completed bootstrap sample: 461 ###"
## [1] "### Completed bootstrap sample: 462 ###"
## [1] "### Completed bootstrap sample: 463 ###"
## [1] "### Completed bootstrap sample: 464 ###"
## [1] "### Completed bootstrap sample: 465 ###"
## [1] "### Completed bootstrap sample: 466 ###"
## [1] "### Completed bootstrap sample: 467 ###"
## [1] "### Completed bootstrap sample: 468 ###"
## [1] "### Completed bootstrap sample: 469 ###"
## [1] "### Completed bootstrap sample: 470 ###"
## [1] "### Completed bootstrap sample: 471 ###"
## [1] "### Completed bootstrap sample: 472 ###"
## [1] "### Completed bootstrap sample: 473 ###"
## [1] "### Completed bootstrap sample: 474 ###"
## [1] "### Completed bootstrap sample: 475 ###"
## [1] "### Completed bootstrap sample: 476 ###"
## [1] "### Completed bootstrap sample: 477 ###"
## [1] "### Completed bootstrap sample: 478 ###"
## [1] "### Completed bootstrap sample: 479 ###"
## [1] "### Completed bootstrap sample: 480 ###"
## [1] "### Completed bootstrap sample: 481 ###"
## [1] "### Completed bootstrap sample: 482 ###"
## [1] "### Completed bootstrap sample: 483 ###"
## [1] "### Completed bootstrap sample: 484 ###"
## [1] "### Completed bootstrap sample: 485 ###"
## [1] "### Completed bootstrap sample: 486 ###"
## [1] "### Completed bootstrap sample: 487 ###"
## [1] "### Completed bootstrap sample: 488 ###"
## [1] "### Completed bootstrap sample: 489 ###"
## [1] "### Completed bootstrap sample: 490 ###"
## [1] "### Completed bootstrap sample: 491 ###"
## [1] "### Completed bootstrap sample: 492 ###"
## [1] "### Completed bootstrap sample: 493 ###"
## [1] "### Completed bootstrap sample: 494 ###"
## [1] "### Completed bootstrap sample: 495 ###"
## [1] "### Completed bootstrap sample: 496 ###"
## [1] "### Completed bootstrap sample: 497 ###"
## [1] "### Completed bootstrap sample: 498 ###"
## [1] "### Completed bootstrap sample: 499 ###"
## [1] "### Completed bootstrap sample: 500 ###"
## [1] "### Completed bootstrap sample: 501 ###"
## [1] "### Completed bootstrap sample: 502 ###"
## [1] "### Completed bootstrap sample: 503 ###"
## [1] "### Completed bootstrap sample: 504 ###"
## [1] "### Completed bootstrap sample: 505 ###"
## [1] "### Completed bootstrap sample: 506 ###"
## [1] "### Completed bootstrap sample: 507 ###"
## [1] "### Completed bootstrap sample: 508 ###"
## [1] "### Completed bootstrap sample: 509 ###"
## [1] "### Completed bootstrap sample: 510 ###"
## [1] "### Completed bootstrap sample: 511 ###"
## [1] "### Completed bootstrap sample: 512 ###"
## [1] "### Completed bootstrap sample: 513 ###"
## [1] "### Completed bootstrap sample: 514 ###"
## [1] "### Completed bootstrap sample: 515 ###"
## [1] "### Completed bootstrap sample: 516 ###"
## [1] "### Completed bootstrap sample: 517 ###"
## [1] "### Completed bootstrap sample: 518 ###"
## [1] "### Completed bootstrap sample: 519 ###"
## [1] "### Completed bootstrap sample: 520 ###"
## [1] "### Completed bootstrap sample: 521 ###"
## [1] "### Completed bootstrap sample: 522 ###"
## [1] "### Completed bootstrap sample: 523 ###"
## [1] "### Completed bootstrap sample: 524 ###"
## [1] "### Completed bootstrap sample: 525 ###"
## [1] "### Completed bootstrap sample: 526 ###"
## [1] "### Completed bootstrap sample: 527 ###"
## [1] "### Completed bootstrap sample: 528 ###"
## [1] "### Completed bootstrap sample: 529 ###"
## [1] "### Completed bootstrap sample: 530 ###"
## [1] "### Completed bootstrap sample: 531 ###"
## [1] "### Completed bootstrap sample: 532 ###"
## [1] "### Completed bootstrap sample: 533 ###"
## [1] "### Completed bootstrap sample: 534 ###"
## [1] "### Completed bootstrap sample: 535 ###"
## [1] "### Completed bootstrap sample: 536 ###"
## [1] "### Completed bootstrap sample: 537 ###"
## [1] "### Completed bootstrap sample: 538 ###"
## [1] "### Completed bootstrap sample: 539 ###"
## [1] "### Completed bootstrap sample: 540 ###"
## [1] "### Completed bootstrap sample: 541 ###"
## [1] "### Completed bootstrap sample: 542 ###"
## [1] "### Completed bootstrap sample: 543 ###"
## [1] "### Completed bootstrap sample: 544 ###"
## [1] "### Completed bootstrap sample: 545 ###"
## [1] "### Completed bootstrap sample: 546 ###"
## [1] "### Completed bootstrap sample: 547 ###"
## [1] "### Completed bootstrap sample: 548 ###"
## [1] "### Completed bootstrap sample: 549 ###"
## [1] "### Completed bootstrap sample: 550 ###"
## [1] "### Completed bootstrap sample: 551 ###"
## [1] "### Completed bootstrap sample: 552 ###"
## [1] "### Completed bootstrap sample: 553 ###"
## [1] "### Completed bootstrap sample: 554 ###"
## [1] "### Completed bootstrap sample: 555 ###"
## [1] "### Completed bootstrap sample: 556 ###"
## [1] "### Completed bootstrap sample: 557 ###"
## [1] "### Completed bootstrap sample: 558 ###"
## [1] "### Completed bootstrap sample: 559 ###"
## [1] "### Completed bootstrap sample: 560 ###"
## [1] "### Completed bootstrap sample: 561 ###"
## [1] "### Completed bootstrap sample: 562 ###"
## [1] "### Completed bootstrap sample: 563 ###"
## [1] "### Completed bootstrap sample: 564 ###"
## [1] "### Completed bootstrap sample: 565 ###"
## [1] "### Completed bootstrap sample: 566 ###"
## [1] "### Completed bootstrap sample: 567 ###"
## [1] "### Completed bootstrap sample: 568 ###"
## [1] "### Completed bootstrap sample: 569 ###"
## [1] "### Completed bootstrap sample: 570 ###"
## [1] "### Completed bootstrap sample: 571 ###"
## [1] "### Completed bootstrap sample: 572 ###"
## [1] "### Completed bootstrap sample: 573 ###"
## [1] "### Completed bootstrap sample: 574 ###"
## [1] "### Completed bootstrap sample: 575 ###"
## [1] "### Completed bootstrap sample: 576 ###"
## [1] "### Completed bootstrap sample: 577 ###"
## [1] "### Completed bootstrap sample: 578 ###"
## [1] "### Completed bootstrap sample: 579 ###"
## [1] "### Completed bootstrap sample: 580 ###"
## [1] "### Completed bootstrap sample: 581 ###"
## [1] "### Completed bootstrap sample: 582 ###"
## [1] "### Completed bootstrap sample: 583 ###"
## [1] "### Completed bootstrap sample: 584 ###"
## [1] "### Completed bootstrap sample: 585 ###"
## [1] "### Completed bootstrap sample: 586 ###"
## [1] "### Completed bootstrap sample: 587 ###"
## [1] "### Completed bootstrap sample: 588 ###"
## [1] "### Completed bootstrap sample: 589 ###"
## [1] "### Completed bootstrap sample: 590 ###"
## [1] "### Completed bootstrap sample: 591 ###"
## [1] "### Completed bootstrap sample: 592 ###"
## [1] "### Completed bootstrap sample: 593 ###"
## [1] "### Completed bootstrap sample: 594 ###"
## [1] "### Completed bootstrap sample: 595 ###"
## [1] "### Completed bootstrap sample: 596 ###"
## [1] "### Completed bootstrap sample: 597 ###"
## [1] "### Completed bootstrap sample: 598 ###"
## [1] "### Completed bootstrap sample: 599 ###"
## [1] "### Completed bootstrap sample: 600 ###"
## [1] "### Completed bootstrap sample: 601 ###"
## [1] "### Completed bootstrap sample: 602 ###"
## [1] "### Completed bootstrap sample: 603 ###"
## [1] "### Completed bootstrap sample: 604 ###"
## [1] "### Completed bootstrap sample: 605 ###"
## [1] "### Completed bootstrap sample: 606 ###"
## [1] "### Completed bootstrap sample: 607 ###"
## [1] "### Completed bootstrap sample: 608 ###"
## [1] "### Completed bootstrap sample: 609 ###"
## [1] "### Completed bootstrap sample: 610 ###"
## [1] "### Completed bootstrap sample: 611 ###"
## [1] "### Completed bootstrap sample: 612 ###"
## [1] "### Completed bootstrap sample: 613 ###"
## [1] "### Completed bootstrap sample: 614 ###"
## [1] "### Completed bootstrap sample: 615 ###"
## [1] "### Completed bootstrap sample: 616 ###"
## [1] "### Completed bootstrap sample: 617 ###"
## [1] "### Completed bootstrap sample: 618 ###"
## [1] "### Completed bootstrap sample: 619 ###"
## [1] "### Completed bootstrap sample: 620 ###"
## [1] "### Completed bootstrap sample: 621 ###"
## [1] "### Completed bootstrap sample: 622 ###"
## [1] "### Completed bootstrap sample: 623 ###"
## [1] "### Completed bootstrap sample: 624 ###"
## [1] "### Completed bootstrap sample: 625 ###"
## [1] "### Completed bootstrap sample: 626 ###"
## [1] "### Completed bootstrap sample: 627 ###"
## [1] "### Completed bootstrap sample: 628 ###"
## [1] "### Completed bootstrap sample: 629 ###"
## [1] "### Completed bootstrap sample: 630 ###"
## [1] "### Completed bootstrap sample: 631 ###"
## [1] "### Completed bootstrap sample: 632 ###"
## [1] "### Completed bootstrap sample: 633 ###"
## [1] "### Completed bootstrap sample: 634 ###"
## [1] "### Completed bootstrap sample: 635 ###"
## [1] "### Completed bootstrap sample: 636 ###"
## [1] "### Completed bootstrap sample: 637 ###"
## [1] "### Completed bootstrap sample: 638 ###"
## [1] "### Completed bootstrap sample: 639 ###"
## [1] "### Completed bootstrap sample: 640 ###"
## [1] "### Completed bootstrap sample: 641 ###"
## [1] "### Completed bootstrap sample: 642 ###"
## [1] "### Completed bootstrap sample: 643 ###"
## [1] "### Completed bootstrap sample: 644 ###"
## [1] "### Completed bootstrap sample: 645 ###"
## [1] "### Completed bootstrap sample: 646 ###"
## [1] "### Completed bootstrap sample: 647 ###"
## [1] "### Completed bootstrap sample: 648 ###"
## [1] "### Completed bootstrap sample: 649 ###"
## [1] "### Completed bootstrap sample: 650 ###"
## [1] "### Completed bootstrap sample: 651 ###"
## [1] "### Completed bootstrap sample: 652 ###"
## [1] "### Completed bootstrap sample: 653 ###"
## [1] "### Completed bootstrap sample: 654 ###"
## [1] "### Completed bootstrap sample: 655 ###"
## [1] "### Completed bootstrap sample: 656 ###"
## [1] "### Completed bootstrap sample: 657 ###"
## [1] "### Completed bootstrap sample: 658 ###"
## [1] "### Completed bootstrap sample: 659 ###"
## [1] "### Completed bootstrap sample: 660 ###"
## [1] "### Completed bootstrap sample: 661 ###"
## [1] "### Completed bootstrap sample: 662 ###"
## [1] "### Completed bootstrap sample: 663 ###"
## [1] "### Completed bootstrap sample: 664 ###"
## [1] "### Completed bootstrap sample: 665 ###"
## [1] "### Completed bootstrap sample: 666 ###"
## [1] "### Completed bootstrap sample: 667 ###"
## [1] "### Completed bootstrap sample: 668 ###"
## [1] "### Completed bootstrap sample: 669 ###"
## [1] "### Completed bootstrap sample: 670 ###"
## [1] "### Completed bootstrap sample: 671 ###"
## [1] "### Completed bootstrap sample: 672 ###"
## [1] "### Completed bootstrap sample: 673 ###"
## [1] "### Completed bootstrap sample: 674 ###"
## [1] "### Completed bootstrap sample: 675 ###"
## [1] "### Completed bootstrap sample: 676 ###"
## [1] "### Completed bootstrap sample: 677 ###"
## [1] "### Completed bootstrap sample: 678 ###"
## [1] "### Completed bootstrap sample: 679 ###"
## [1] "### Completed bootstrap sample: 680 ###"
## [1] "### Completed bootstrap sample: 681 ###"
## [1] "### Completed bootstrap sample: 682 ###"
## [1] "### Completed bootstrap sample: 683 ###"
## [1] "### Completed bootstrap sample: 684 ###"
## [1] "### Completed bootstrap sample: 685 ###"
## [1] "### Completed bootstrap sample: 686 ###"
## [1] "### Completed bootstrap sample: 687 ###"
## [1] "### Completed bootstrap sample: 688 ###"
## [1] "### Completed bootstrap sample: 689 ###"
## [1] "### Completed bootstrap sample: 690 ###"
## [1] "### Completed bootstrap sample: 691 ###"
## [1] "### Completed bootstrap sample: 692 ###"
## [1] "### Completed bootstrap sample: 693 ###"
## [1] "### Completed bootstrap sample: 694 ###"
## [1] "### Completed bootstrap sample: 695 ###"
## [1] "### Completed bootstrap sample: 696 ###"
## [1] "### Completed bootstrap sample: 697 ###"
## [1] "### Completed bootstrap sample: 698 ###"
## [1] "### Completed bootstrap sample: 699 ###"
## [1] "### Completed bootstrap sample: 700 ###"
## [1] "### Completed bootstrap sample: 701 ###"
## [1] "### Completed bootstrap sample: 702 ###"
## [1] "### Completed bootstrap sample: 703 ###"
## [1] "### Completed bootstrap sample: 704 ###"
## [1] "### Completed bootstrap sample: 705 ###"
## [1] "### Completed bootstrap sample: 706 ###"
## [1] "### Completed bootstrap sample: 707 ###"
## [1] "### Completed bootstrap sample: 708 ###"
## [1] "### Completed bootstrap sample: 709 ###"
## [1] "### Completed bootstrap sample: 710 ###"
## [1] "### Completed bootstrap sample: 711 ###"
## [1] "### Completed bootstrap sample: 712 ###"
## [1] "### Completed bootstrap sample: 713 ###"
## [1] "### Completed bootstrap sample: 714 ###"
## [1] "### Completed bootstrap sample: 715 ###"
## [1] "### Completed bootstrap sample: 716 ###"
## [1] "### Completed bootstrap sample: 717 ###"
## [1] "### Completed bootstrap sample: 718 ###"
## [1] "### Completed bootstrap sample: 719 ###"
## [1] "### Completed bootstrap sample: 720 ###"
## [1] "### Completed bootstrap sample: 721 ###"
## [1] "### Completed bootstrap sample: 722 ###"
## [1] "### Completed bootstrap sample: 723 ###"
## [1] "### Completed bootstrap sample: 724 ###"
## [1] "### Completed bootstrap sample: 725 ###"
## [1] "### Completed bootstrap sample: 726 ###"
## [1] "### Completed bootstrap sample: 727 ###"
## [1] "### Completed bootstrap sample: 728 ###"
## [1] "### Completed bootstrap sample: 729 ###"
## [1] "### Completed bootstrap sample: 730 ###"
## [1] "### Completed bootstrap sample: 731 ###"
## [1] "### Completed bootstrap sample: 732 ###"
## [1] "### Completed bootstrap sample: 733 ###"
## [1] "### Completed bootstrap sample: 734 ###"
## [1] "### Completed bootstrap sample: 735 ###"
## [1] "### Completed bootstrap sample: 736 ###"
## [1] "### Completed bootstrap sample: 737 ###"
## [1] "### Completed bootstrap sample: 738 ###"
## [1] "### Completed bootstrap sample: 739 ###"
## [1] "### Completed bootstrap sample: 740 ###"
## [1] "### Completed bootstrap sample: 741 ###"
## [1] "### Completed bootstrap sample: 742 ###"
## [1] "### Completed bootstrap sample: 743 ###"
## [1] "### Completed bootstrap sample: 744 ###"
## [1] "### Completed bootstrap sample: 745 ###"
## [1] "### Completed bootstrap sample: 746 ###"
## [1] "### Completed bootstrap sample: 747 ###"
## [1] "### Completed bootstrap sample: 748 ###"
## [1] "### Completed bootstrap sample: 749 ###"
## [1] "### Completed bootstrap sample: 750 ###"
## [1] "### Completed bootstrap sample: 751 ###"
## [1] "### Completed bootstrap sample: 752 ###"
## [1] "### Completed bootstrap sample: 753 ###"
## [1] "### Completed bootstrap sample: 754 ###"
## [1] "### Completed bootstrap sample: 755 ###"
## [1] "### Completed bootstrap sample: 756 ###"
## [1] "### Completed bootstrap sample: 757 ###"
## [1] "### Completed bootstrap sample: 758 ###"
## [1] "### Completed bootstrap sample: 759 ###"
## [1] "### Completed bootstrap sample: 760 ###"
## [1] "### Completed bootstrap sample: 761 ###"
## [1] "### Completed bootstrap sample: 762 ###"
## [1] "### Completed bootstrap sample: 763 ###"
## [1] "### Completed bootstrap sample: 764 ###"
## [1] "### Completed bootstrap sample: 765 ###"
## [1] "### Completed bootstrap sample: 766 ###"
## [1] "### Completed bootstrap sample: 767 ###"
## [1] "### Completed bootstrap sample: 768 ###"
## [1] "### Completed bootstrap sample: 769 ###"
## [1] "### Completed bootstrap sample: 770 ###"
## [1] "### Completed bootstrap sample: 771 ###"
## [1] "### Completed bootstrap sample: 772 ###"
## [1] "### Completed bootstrap sample: 773 ###"
## [1] "### Completed bootstrap sample: 774 ###"
## [1] "### Completed bootstrap sample: 775 ###"
## [1] "### Completed bootstrap sample: 776 ###"
## [1] "### Completed bootstrap sample: 777 ###"
## [1] "### Completed bootstrap sample: 778 ###"
## [1] "### Completed bootstrap sample: 779 ###"
## [1] "### Completed bootstrap sample: 780 ###"
## [1] "### Completed bootstrap sample: 781 ###"
## [1] "### Completed bootstrap sample: 782 ###"
## [1] "### Completed bootstrap sample: 783 ###"
## [1] "### Completed bootstrap sample: 784 ###"
## [1] "### Completed bootstrap sample: 785 ###"
## [1] "### Completed bootstrap sample: 786 ###"
## [1] "### Completed bootstrap sample: 787 ###"
## [1] "### Completed bootstrap sample: 788 ###"
## [1] "### Completed bootstrap sample: 789 ###"
## [1] "### Completed bootstrap sample: 790 ###"
## [1] "### Completed bootstrap sample: 791 ###"
## [1] "### Completed bootstrap sample: 792 ###"
## [1] "### Completed bootstrap sample: 793 ###"
## [1] "### Completed bootstrap sample: 794 ###"
## [1] "### Completed bootstrap sample: 795 ###"
## [1] "### Completed bootstrap sample: 796 ###"
## [1] "### Completed bootstrap sample: 797 ###"
## [1] "### Completed bootstrap sample: 798 ###"
## [1] "### Completed bootstrap sample: 799 ###"
## [1] "### Completed bootstrap sample: 800 ###"
## [1] "### Completed bootstrap sample: 801 ###"
## [1] "### Completed bootstrap sample: 802 ###"
## [1] "### Completed bootstrap sample: 803 ###"
## [1] "### Completed bootstrap sample: 804 ###"
## [1] "### Completed bootstrap sample: 805 ###"
## [1] "### Completed bootstrap sample: 806 ###"
## [1] "### Completed bootstrap sample: 807 ###"
## [1] "### Completed bootstrap sample: 808 ###"
## [1] "### Completed bootstrap sample: 809 ###"
## [1] "### Completed bootstrap sample: 810 ###"
## [1] "### Completed bootstrap sample: 811 ###"
## [1] "### Completed bootstrap sample: 812 ###"
## [1] "### Completed bootstrap sample: 813 ###"
## [1] "### Completed bootstrap sample: 814 ###"
## [1] "### Completed bootstrap sample: 815 ###"
## [1] "### Completed bootstrap sample: 816 ###"
## [1] "### Completed bootstrap sample: 817 ###"
## [1] "### Completed bootstrap sample: 818 ###"
## [1] "### Completed bootstrap sample: 819 ###"
## [1] "### Completed bootstrap sample: 820 ###"
## [1] "### Completed bootstrap sample: 821 ###"
## [1] "### Completed bootstrap sample: 822 ###"
## [1] "### Completed bootstrap sample: 823 ###"
## [1] "### Completed bootstrap sample: 824 ###"
## [1] "### Completed bootstrap sample: 825 ###"
## [1] "### Completed bootstrap sample: 826 ###"
## [1] "### Completed bootstrap sample: 827 ###"
## [1] "### Completed bootstrap sample: 828 ###"
## [1] "### Completed bootstrap sample: 829 ###"
## [1] "### Completed bootstrap sample: 830 ###"
## [1] "### Completed bootstrap sample: 831 ###"
## [1] "### Completed bootstrap sample: 832 ###"
## [1] "### Completed bootstrap sample: 833 ###"
## [1] "### Completed bootstrap sample: 834 ###"
## [1] "### Completed bootstrap sample: 835 ###"
## [1] "### Completed bootstrap sample: 836 ###"
## [1] "### Completed bootstrap sample: 837 ###"
## [1] "### Completed bootstrap sample: 838 ###"
## [1] "### Completed bootstrap sample: 839 ###"
## [1] "### Completed bootstrap sample: 840 ###"
## [1] "### Completed bootstrap sample: 841 ###"
## [1] "### Completed bootstrap sample: 842 ###"
## [1] "### Completed bootstrap sample: 843 ###"
## [1] "### Completed bootstrap sample: 844 ###"
## [1] "### Completed bootstrap sample: 845 ###"
## [1] "### Completed bootstrap sample: 846 ###"
## [1] "### Completed bootstrap sample: 847 ###"
## [1] "### Completed bootstrap sample: 848 ###"
## [1] "### Completed bootstrap sample: 849 ###"
## [1] "### Completed bootstrap sample: 850 ###"
## [1] "### Completed bootstrap sample: 851 ###"
## [1] "### Completed bootstrap sample: 852 ###"
## [1] "### Completed bootstrap sample: 853 ###"
## [1] "### Completed bootstrap sample: 854 ###"
## [1] "### Completed bootstrap sample: 855 ###"
## [1] "### Completed bootstrap sample: 856 ###"
## [1] "### Completed bootstrap sample: 857 ###"
## [1] "### Completed bootstrap sample: 858 ###"
## [1] "### Completed bootstrap sample: 859 ###"
## [1] "### Completed bootstrap sample: 860 ###"
## [1] "### Completed bootstrap sample: 861 ###"
## [1] "### Completed bootstrap sample: 862 ###"
## [1] "### Completed bootstrap sample: 863 ###"
## [1] "### Completed bootstrap sample: 864 ###"
## [1] "### Completed bootstrap sample: 865 ###"
## [1] "### Completed bootstrap sample: 866 ###"
## [1] "### Completed bootstrap sample: 867 ###"
## [1] "### Completed bootstrap sample: 868 ###"
## [1] "### Completed bootstrap sample: 869 ###"
## [1] "### Completed bootstrap sample: 870 ###"
## [1] "### Completed bootstrap sample: 871 ###"
## [1] "### Completed bootstrap sample: 872 ###"
## [1] "### Completed bootstrap sample: 873 ###"
## [1] "### Completed bootstrap sample: 874 ###"
## [1] "### Completed bootstrap sample: 875 ###"
## [1] "### Completed bootstrap sample: 876 ###"
## [1] "### Completed bootstrap sample: 877 ###"
## [1] "### Completed bootstrap sample: 878 ###"
## [1] "### Completed bootstrap sample: 879 ###"
## [1] "### Completed bootstrap sample: 880 ###"
## [1] "### Completed bootstrap sample: 881 ###"
## [1] "### Completed bootstrap sample: 882 ###"
## [1] "### Completed bootstrap sample: 883 ###"
## [1] "### Completed bootstrap sample: 884 ###"
## [1] "### Completed bootstrap sample: 885 ###"
## [1] "### Completed bootstrap sample: 886 ###"
## [1] "### Completed bootstrap sample: 887 ###"
## [1] "### Completed bootstrap sample: 888 ###"
## [1] "### Completed bootstrap sample: 889 ###"
## [1] "### Completed bootstrap sample: 890 ###"
## [1] "### Completed bootstrap sample: 891 ###"
## [1] "### Completed bootstrap sample: 892 ###"
## [1] "### Completed bootstrap sample: 893 ###"
## [1] "### Completed bootstrap sample: 894 ###"
## [1] "### Completed bootstrap sample: 895 ###"
## [1] "### Completed bootstrap sample: 896 ###"
## [1] "### Completed bootstrap sample: 897 ###"
## [1] "### Completed bootstrap sample: 898 ###"
## [1] "### Completed bootstrap sample: 899 ###"
## [1] "### Completed bootstrap sample: 900 ###"
## [1] "### Completed bootstrap sample: 901 ###"
## [1] "### Completed bootstrap sample: 902 ###"
## [1] "### Completed bootstrap sample: 903 ###"
## [1] "### Completed bootstrap sample: 904 ###"
## [1] "### Completed bootstrap sample: 905 ###"
## [1] "### Completed bootstrap sample: 906 ###"
## [1] "### Completed bootstrap sample: 907 ###"
## [1] "### Completed bootstrap sample: 908 ###"
## [1] "### Completed bootstrap sample: 909 ###"
## [1] "### Completed bootstrap sample: 910 ###"
## [1] "### Completed bootstrap sample: 911 ###"
## [1] "### Completed bootstrap sample: 912 ###"
## [1] "### Completed bootstrap sample: 913 ###"
## [1] "### Completed bootstrap sample: 914 ###"
## [1] "### Completed bootstrap sample: 915 ###"
## [1] "### Completed bootstrap sample: 916 ###"
## [1] "### Completed bootstrap sample: 917 ###"
## [1] "### Completed bootstrap sample: 918 ###"
## [1] "### Completed bootstrap sample: 919 ###"
## [1] "### Completed bootstrap sample: 920 ###"
## [1] "### Completed bootstrap sample: 921 ###"
## [1] "### Completed bootstrap sample: 922 ###"
## [1] "### Completed bootstrap sample: 923 ###"
## [1] "### Completed bootstrap sample: 924 ###"
## [1] "### Completed bootstrap sample: 925 ###"
## [1] "### Completed bootstrap sample: 926 ###"
## [1] "### Completed bootstrap sample: 927 ###"
## [1] "### Completed bootstrap sample: 928 ###"
## [1] "### Completed bootstrap sample: 929 ###"
## [1] "### Completed bootstrap sample: 930 ###"
## [1] "### Completed bootstrap sample: 931 ###"
## [1] "### Completed bootstrap sample: 932 ###"
## [1] "### Completed bootstrap sample: 933 ###"
## [1] "### Completed bootstrap sample: 934 ###"
## [1] "### Completed bootstrap sample: 935 ###"
## [1] "### Completed bootstrap sample: 936 ###"
## [1] "### Completed bootstrap sample: 937 ###"
## [1] "### Completed bootstrap sample: 938 ###"
## [1] "### Completed bootstrap sample: 939 ###"
## [1] "### Completed bootstrap sample: 940 ###"
## [1] "### Completed bootstrap sample: 941 ###"
## [1] "### Completed bootstrap sample: 942 ###"
## [1] "### Completed bootstrap sample: 943 ###"
## [1] "### Completed bootstrap sample: 944 ###"
## [1] "### Completed bootstrap sample: 945 ###"
## [1] "### Completed bootstrap sample: 946 ###"
## [1] "### Completed bootstrap sample: 947 ###"
## [1] "### Completed bootstrap sample: 948 ###"
## [1] "### Completed bootstrap sample: 949 ###"
## [1] "### Completed bootstrap sample: 950 ###"
## [1] "### Completed bootstrap sample: 951 ###"
## [1] "### Completed bootstrap sample: 952 ###"
## [1] "### Completed bootstrap sample: 953 ###"
## [1] "### Completed bootstrap sample: 954 ###"
## [1] "### Completed bootstrap sample: 955 ###"
## [1] "### Completed bootstrap sample: 956 ###"
## [1] "### Completed bootstrap sample: 957 ###"
## [1] "### Completed bootstrap sample: 958 ###"
## [1] "### Completed bootstrap sample: 959 ###"
## [1] "### Completed bootstrap sample: 960 ###"
## [1] "### Completed bootstrap sample: 961 ###"
## [1] "### Completed bootstrap sample: 962 ###"
## [1] "### Completed bootstrap sample: 963 ###"
## [1] "### Completed bootstrap sample: 964 ###"
## [1] "### Completed bootstrap sample: 965 ###"
## [1] "### Completed bootstrap sample: 966 ###"
## [1] "### Completed bootstrap sample: 967 ###"
## [1] "### Completed bootstrap sample: 968 ###"
## [1] "### Completed bootstrap sample: 969 ###"
## [1] "### Completed bootstrap sample: 970 ###"
## [1] "### Completed bootstrap sample: 971 ###"
## [1] "### Completed bootstrap sample: 972 ###"
## [1] "### Completed bootstrap sample: 973 ###"
## [1] "### Completed bootstrap sample: 974 ###"
## [1] "### Completed bootstrap sample: 975 ###"
## [1] "### Completed bootstrap sample: 976 ###"
## [1] "### Completed bootstrap sample: 977 ###"
## [1] "### Completed bootstrap sample: 978 ###"
## [1] "### Completed bootstrap sample: 979 ###"
## [1] "### Completed bootstrap sample: 980 ###"
## [1] "### Completed bootstrap sample: 981 ###"
## [1] "### Completed bootstrap sample: 982 ###"
## [1] "### Completed bootstrap sample: 983 ###"
## [1] "### Completed bootstrap sample: 984 ###"
## [1] "### Completed bootstrap sample: 985 ###"
## [1] "### Completed bootstrap sample: 986 ###"
## [1] "### Completed bootstrap sample: 987 ###"
## [1] "### Completed bootstrap sample: 988 ###"
## [1] "### Completed bootstrap sample: 989 ###"
## [1] "### Completed bootstrap sample: 990 ###"
## [1] "### Completed bootstrap sample: 991 ###"
## [1] "### Completed bootstrap sample: 992 ###"
## [1] "### Completed bootstrap sample: 993 ###"
## [1] "### Completed bootstrap sample: 994 ###"
## [1] "### Completed bootstrap sample: 995 ###"
## [1] "### Completed bootstrap sample: 996 ###"
## [1] "### Completed bootstrap sample: 997 ###"
## [1] "### Completed bootstrap sample: 998 ###"
## [1] "### Completed bootstrap sample: 999 ###"
## [1] "### Completed bootstrap sample: 1000 ###"
## [1] "### Completed bootstrap sample: 1001 ###"
## [1] "### Completed bootstrap sample: 1002 ###"
## [1] "### Completed bootstrap sample: 1003 ###"
## [1] "### Completed bootstrap sample: 1004 ###"
## [1] "### Completed bootstrap sample: 1005 ###"
## [1] "### Completed bootstrap sample: 1006 ###"
## [1] "### Completed bootstrap sample: 1007 ###"
## [1] "### Completed bootstrap sample: 1008 ###"
## [1] "### Completed bootstrap sample: 1009 ###"
## [1] "### Completed bootstrap sample: 1010 ###"
## [1] "### Completed bootstrap sample: 1011 ###"
## [1] "### Completed bootstrap sample: 1012 ###"
## [1] "### Completed bootstrap sample: 1013 ###"
## [1] "### Completed bootstrap sample: 1014 ###"
## [1] "### Completed bootstrap sample: 1015 ###"
## [1] "### Completed bootstrap sample: 1016 ###"
## [1] "### Completed bootstrap sample: 1017 ###"
## [1] "### Completed bootstrap sample: 1018 ###"
## [1] "### Completed bootstrap sample: 1019 ###"
## [1] "### Completed bootstrap sample: 1020 ###"
## [1] "### Completed bootstrap sample: 1021 ###"
nsvm3
## Confusion Matrix and Statistics
##
##
## 0 1 2 3 4 5
## 0 49 11 2 0 3 0
## 1 3 217 28 15 32 1
## 2 3 36 186 4 18 0
## 3 1 29 10 46 15 0
## 4 0 38 18 9 226 1
## 5 0 2 1 1 8 5
##
## Overall Statistics
##
## Accuracy : 0.7161
## 95% CI : (0.6873, 0.7436)
## No Information Rate : 0.3271
## P-Value [Acc > NIR] : < 2.2e-16
##
## Kappa : 0.6217
##
## Mcnemar's Test P-Value : NA
##
## Statistics by Class:
##
## Class: 0 Class: 1 Class: 2 Class: 3 Class: 4 Class: 5
## Sensitivity 0.87500 0.6517 0.7592 0.61333 0.7483 0.714286
## Specificity 0.98337 0.8847 0.9211 0.94168 0.9078 0.988131
## Pos Pred Value 0.75385 0.7331 0.7530 0.45545 0.7740 0.294118
## Neg Pred Value 0.99265 0.8393 0.9235 0.96838 0.8953 0.998002
## Precision 0.75385 0.7331 0.7530 0.45545 0.7740 0.294118
## Recall 0.87500 0.6517 0.7592 0.61333 0.7483 0.714286
## F1 0.80992 0.6900 0.7561 0.52273 0.7609 0.416667
## Prevalence 0.05501 0.3271 0.2407 0.07367 0.2967 0.006876
## Detection Rate 0.04813 0.2132 0.1827 0.04519 0.2220 0.004912
## Detection Prevalence 0.06385 0.2908 0.2426 0.09921 0.2868 0.016699
## Balanced Accuracy 0.92918 0.7682 0.8401 0.77750 0.8281 0.851208