Neural Network Image Classification
Satyo Pradana
2023-11-28
Dataset
Ini adalah kumpulan data gambar kartu remi. Semua gambar berukuran 224 X 224 X 3 dalam format jpg. Semua gambar dalam kumpulan data telah dipotong sehingga hanya gambar dari satu kartu yang ditampilkan dan kartu tersebut menempati lebih dari 50% piksel dalam gambar. Terdapat 7624 gambar latih, 265 gambar uji, dan 265 gambar validasi. Direktori pelatihan, pengujian, dan validasi dipartisi menjadi 53 subdirektori, satu untuk masing-masing dari 53 jenis kartu. Kumpulan data juga menyertakan file csv yang dapat digunakan untuk memuat kumpulan data.
library(tidyr)
library(dplyr)##
## Attaching package: 'dplyr'## The following objects are masked from 'package:stats':
##
## filter, lag## The following objects are masked from 'package:base':
##
## intersect, setdiff, setequal, unionlibrary(keras)## Warning: package 'keras' was built under R version 4.3.2library(caret)## Loading required package: ggplot2## Loading required package: latticelibrary(imager)## Warning: package 'imager' was built under R version 4.3.2## Loading required package: magrittr##
## Attaching package: 'magrittr'## The following object is masked from 'package:tidyr':
##
## extract##
## Attaching package: 'imager'## The following object is masked from 'package:magrittr':
##
## add## The following object is masked from 'package:dplyr':
##
## where## The following object is masked from 'package:tidyr':
##
## fill## The following objects are masked from 'package:stats':
##
## convolve, spectrum## The following object is masked from 'package:graphics':
##
## frame## The following object is masked from 'package:base':
##
## save.imagelibrary(base)
library(purrr)##
## Attaching package: 'purrr'## The following object is masked from 'package:magrittr':
##
## set_names## The following object is masked from 'package:caret':
##
## liftlibrary(ggplot2)
library(stringr)##
## Attaching package: 'stringr'## The following object is masked from 'package:imager':
##
## boundaryuse_condaenv("r-tensorflow")
options(scipen = 999)Input Dataset
folder_list <- list.files("image cla/train/")
folder_list## [1] "ace of clubs" "ace of diamonds" "ace of hearts"
## [4] "ace of spades" "eight of clubs" "eight of diamonds"
## [7] "eight of hearts" "eight of spades" "five of clubs"
## [10] "five of diamonds" "five of hearts" "five of spades"
## [13] "four of clubs" "four of diamonds" "four of hearts"
## [16] "four of spades" "jack of clubs" "jack of diamonds"
## [19] "jack of hearts" "jack of spades" "joker"
## [22] "king of clubs" "king of diamonds" "king of hearts"
## [25] "king of spades" "nine of clubs" "nine of diamonds"
## [28] "nine of hearts" "nine of spades" "queen of clubs"
## [31] "queen of diamonds" "queen of hearts" "queen of spades"
## [34] "seven of clubs" "seven of diamonds" "seven of hearts"
## [37] "seven of spades" "six of clubs" "six of diamonds"
## [40] "six of hearts" "six of spades" "ten of clubs"
## [43] "ten of diamonds" "ten of hearts" "ten of spades"
## [46] "three of clubs" "three of diamonds" "three of hearts"
## [49] "three of spades" "two of clubs" "two of diamonds"
## [52] "two of hearts" "two of spades"folder_path <- paste0("image cla/train/")
folder_path## [1] "image cla/train/"file_name <- map(folder_path,
function(x) paste0(x,list.files(x))
) %>%
unlist()
head(file_name)## [1] "image cla/train/ace of clubs" "image cla/train/ace of diamonds"
## [3] "image cla/train/ace of hearts" "image cla/train/ace of spades"
## [5] "image cla/train/eight of clubs" "image cla/train/eight of diamonds"load image
# function for acquiring width and height of an image
get_dim <- function(x){
img <- load.image(x)
df_img <- data.frame(height(img),
width(img),
file_name =x
)
return(df_img)
}# Desired height and width of images
target_size <- c(64, 64)
# Batch size for training the model
batch_size <- 256Men-generate image dari data train, validasi serta test dengan menormalisasi pixel, melakukan perputaran gambar, men-zoom serta pergeseran gambar.
# Image Generator
train_data_gen <- image_data_generator(rescale = 1/255,
horizontal_flip = T,
width_shift_range = 0.2,
height_shift_range = 0.2,
zoom_range = 0.2,
brightness_range = c(1,2),
fill_mode = "nearest")
valid_data_gen <- image_data_generator(rescale = 1/255)
test_data_gen <- image_data_generator(rescale = 1/255)Men-generate dataset dengan masing-masing folder directory untuk persiapan model
# Training Dataset
train_image_array_gen <-
flow_images_from_directory(directory = "image cla/train/",
target_size = target_size,
color_mode = "rgb", # use RGB color
batch_size = batch_size ,
seed = 123, # set random seed
generator = train_data_gen,
class_mode = "categorical" )## Found 7624 images belonging to 53 classes.# Validation Dataset
val_image_array_gen <- flow_images_from_directory(directory = "image cla/valid/",
target_size = target_size,
color_mode = "rgb",
batch_size = batch_size ,
seed = 123,
generator = valid_data_gen,
class_mode = "categorical"
)## Found 265 images belonging to 53 classes.# Testing Dataset
test_image_array_gen <- flow_images_from_directory(directory = "image cla/test/",
target_size = target_size,
color_mode = "rgb",
batch_size = batch_size ,
seed = 123,
generator = test_data_gen,
class_mode = "categorical"
)## Found 265 images belonging to 53 classes.Menentukan jumlah sample yang akan dijadikan data train, validasi dan test serta melihat perbadningan ratio dari ke 53 class
#Number of training samples
train_samples <- train_image_array_gen$n
# Number of validation samples
valid_samples <- val_image_array_gen$n
# Number of testing samples
test_samples <-test_image_array_gen$n
# Number of target classes/categories
output_n <- n_distinct(train_image_array_gen$classes)
# Get the class proportion
table("\nFrequency" = factor(train_image_array_gen$classes)
) %>%
prop.table()##
## Frequency
## 0 1 2 3 4 5 6
## 0.01573977 0.01692025 0.02242917 0.02374082 0.01810073 0.02085519 0.01993704
## 7 8 9 10 11 12 13
## 0.01770724 0.01967471 0.01810073 0.01783841 0.02072403 0.02059286 0.01495278
## 14 15 16 17 18 19 20
## 0.02019937 0.01836306 0.02242917 0.02098636 0.02203568 0.02256034 0.01508395
## 21 22 23 24 25 26 27
## 0.01678909 0.01770724 0.01639559 0.01980588 0.01626443 0.01692025 0.01744491
## 28 29 30 31 32 33 34
## 0.02019937 0.02111752 0.02137985 0.01823190 0.02124869 0.01416579 0.01626443
## 35 36 37 38 39 40 41
## 0.01875656 0.02164218 0.01993704 0.01823190 0.01665792 0.02072403 0.01849423
## 42 43 44 45 46 47 48
## 0.01980588 0.01692025 0.02072403 0.01652676 0.02006821 0.01482162 0.01862539
## 49 50 51 52
## 0.01705142 0.01744491 0.02033054 0.02033054Dilakukan perhitungan ratio dari 53 class maka datanya bisa dikatakan balance
c(target_size, 53)## [1] 64 64 53Melakukan persiapan model untuk training menggunakan CNN Model
#Set Initial Random Weight
tensorflow::tf$random$set_seed(123)
model <- keras_model_sequential(name = "CNN_Model") %>%
# Convolution Layer 1
layer_conv_2d(filters = 64,
kernel_size = c(3,3),
padding = "same",
activation = "relu",
input_shape = c(target_size, 3)
) %>%
# Max Pooling Layer 1
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 2
layer_conv_2d(filters = 64,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 2
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 3
layer_conv_2d(filters = 128,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 3
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Flattening Layer
layer_flatten() %>%
# Dense Layer1
layer_dense(units = 128,
activation = "relu") %>%
# Output Layer
layer_dense(units = output_n,
activation = "softmax",
name = "Output")
model## Model: "CNN_Model"
## ________________________________________________________________________________
## Layer (type) Output Shape Param #
## ================================================================================
## conv2d_2 (Conv2D) (None, 64, 64, 64) 1792
## max_pooling2d_2 (MaxPooling2D) (None, 32, 32, 64) 0
## conv2d_1 (Conv2D) (None, 32, 32, 64) 36928
## max_pooling2d_1 (MaxPooling2D) (None, 16, 16, 64) 0
## conv2d (Conv2D) (None, 16, 16, 128) 73856
## max_pooling2d (MaxPooling2D) (None, 8, 8, 128) 0
## flatten (Flatten) (None, 8192) 0
## dense (Dense) (None, 128) 1048704
## Output (Dense) (None, 53) 6837
## ================================================================================
## Total params: 1,168,117
## Trainable params: 1,168,117
## Non-trainable params: 0
## ________________________________________________________________________________mengcompile model dengan menggunakan optimizer adam serta metrics accuracy
model %>%
compile(
loss = "categorical_crossentropy",
optimizer = optimizer_adam(learning_rate = 0.001),
metrics = "accuracy"
)melakukan fitting / perhitungan model dengan menggunakan 10 epoch
# Fit data into model
history <- model %>%
fit(train_image_array_gen,
# training epochs
steps_per_epoch = as.integer(train_samples / batch_size),
epochs = 10,
# validation data
validation_data = val_image_array_gen,
validation_steps = as.integer(valid_samples / batch_size)
)## Epoch 1/10
## 29/29 - 31s - loss: 3.9446 - accuracy: 0.0252 - val_loss: 3.8420 - val_accuracy: 0.0391 - 31s/epoch - 1s/step
## Epoch 2/10
## 29/29 - 40s - loss: 3.5987 - accuracy: 0.0666 - val_loss: 3.2536 - val_accuracy: 0.1055 - 40s/epoch - 1s/step
## Epoch 3/10
## 29/29 - 61s - loss: 3.2184 - accuracy: 0.1185 - val_loss: 2.9511 - val_accuracy: 0.1445 - 61s/epoch - 2s/step
## Epoch 4/10
## 29/29 - 49s - loss: 3.0333 - accuracy: 0.1516 - val_loss: 2.7371 - val_accuracy: 0.2188 - 49s/epoch - 2s/step
## Epoch 5/10
## 29/29 - 49s - loss: 2.9190 - accuracy: 0.1716 - val_loss: 2.7359 - val_accuracy: 0.2031 - 49s/epoch - 2s/step
## Epoch 6/10
## 29/29 - 57s - loss: 2.8073 - accuracy: 0.2101 - val_loss: 2.4262 - val_accuracy: 0.2812 - 57s/epoch - 2s/step
## Epoch 7/10
## 29/29 - 58s - loss: 2.7216 - accuracy: 0.2172 - val_loss: 2.4484 - val_accuracy: 0.2930 - 58s/epoch - 2s/step
## Epoch 8/10
## 29/29 - 56s - loss: 2.6286 - accuracy: 0.2427 - val_loss: 2.3408 - val_accuracy: 0.3125 - 56s/epoch - 2s/step
## Epoch 9/10
## 29/29 - 63s - loss: 2.5558 - accuracy: 0.2655 - val_loss: 2.2909 - val_accuracy: 0.3203 - 63s/epoch - 2s/step
## Epoch 10/10
## 29/29 - 49s - loss: 2.4797 - accuracy: 0.2858 - val_loss: 2.1736 - val_accuracy: 0.3555 - 49s/epoch - 2s/stepplot(history)
Insight: hasil dari accuracy dan loss berdasarkan data train dan
validasi kurang memuaskan, hal ini sangat dimungkinkan karena dataset
yang digunakan memiliki pixel yang cukup besar, sehingga membutuhkan
memory yang lebih besar dan render time yang lebih lama. sehingga beban
processing harus diperhitungkan
save_model_tf(model,"model")model_load <- load_model_tf("model") val_data <- data.frame(file_name = paste0("image cla/valid/",
val_image_array_gen$filenames)) %>%
mutate(class = str_extract(file_name,"ace of clubs|ace of hearts|ace of spades|ace of diamonds|two of hearts|two of spades|two of clubs|two of diamonds|three of hearts|three of spades|three of diamonds|three of clubs|four of spades|four of hearts|four of diamonds|four of clubs|five of spades|five of hearts|five of diamonds|five of clubs|six of spades|six of hearts|six of diamonds|six of clubs|seven of spades|seven of hearts|seven of diamonds|seven of clubs|eight of spades|eight of hearts|eight of diamonds|eight of clubs|nine of spades|nine of hearts|nine of diamonds|nine of clubs|ten of spades|ten of hearts|ten of diamonds|ten of clubs| jack of spades|jack of hearts|jack of diamonds|jack of clubs|queen of spades|queen of hearts|queen of diamonds|queen of clubs|king of spades|king of hearts|king of diamonds|king of clubs|joker|jack of spades"))
tail(val_data, 10)head(val_data,10)val_datacek file name dengan class label yang ditentukan
# Function to convert image to array
image_prep <- function(x) {
arrays <- lapply(x, function(path) {
img <- image_load(path, target_size = target_size
)
x <- image_to_array(img)
x <- array_reshape(x, c(1, dim(x)))
x <- x/255 # rescale image pixel
})
do.call(abind::abind, c(arrays, list(along = 1)))
}val_x <- image_prep(val_data$file_name)
# Check dimension of testing data set
dim(val_x)## [1] 265 64 64 3pred_val <- predict(model, val_x) %>% k_argmax() %>% as.array()## 9/9 - 0s - 409ms/epoch - 45ms/steppred_val## [1] 3 0 19 0 0 2 1 27 37 1 1 1 3 1 2 3 3 3 3 40 41 25 44 41 33
## [26] 35 5 35 38 35 35 35 22 3 35 7 7 44 7 36 8 33 3 8 8 9 3 10 10 35
## [51] 10 10 35 10 5 7 8 11 11 8 12 3 7 12 1 14 13 35 10 6 14 14 10 14 6
## [76] 12 15 36 36 15 16 16 19 16 16 17 22 19 19 22 30 19 18 16 23 16 19 19 16 19
## [101] 22 29 16 20 16 16 24 16 16 16 22 22 16 22 19 16 30 18 22 17 16 21 24 16 16
## [126] 28 33 25 8 41 26 43 27 5 44 27 5 30 27 17 28 28 44 41 25 21 16 29 24 16
## [151] 16 22 17 22 22 22 16 22 17 16 24 16 16 16 16 33 40 3 29 33 35 35 39 43 10
## [176] 39 35 35 6 35 36 36 37 19 36 37 25 40 25 37 39 30 39 39 6 43 39 39 29 6
## [201] 40 19 40 40 36 41 3 41 41 41 5 43 19 43 43 43 43 43 22 19 28 44 44 44 41
## [226] 30 8 45 45 37 5 46 46 1 5 47 46 47 47 47 48 48 16 48 48 45 8 52 8 52
## [251] 22 50 1 2 14 51 16 51 10 3 11 3 52 52 48# Convert encoding to label
decode <- function(x){
case_when(x == 0 ~ "ace of clubs",
x == 1 ~ "ace of diamonds",
x == 2 ~"ace of hearts",
x == 3 ~"ace of spades",
x == 4 ~"eight of clubs",
x == 5~"eight of diamonds",
x == 6~"eight of hearts",
x==7~"eight of spades",
x==8~"five of clubs",
x==9~"five of diamonds",
x==10~"five of hearts",
x==11~"five of spades",
x==12~"four of clubs",
x==13~"four of diamonds",
x==14~"four of hearts",
x==15~"four of spades",
x==16~"jack of clubs",
x==17~"jack of diamonds",
x==18~"jack of hearts",
x==19~"jack of spades",
x==20~"joker",
x==21~"king of clubs",
x==22~"king of diamonds",
x==23~"king of heart",
x==24~"king of spades",
x==25~"queen of clubs",
x==26~"queen of diamonds",
x==27~"queen of hearts",
x==28~"queen of spades",
x==29~"two of spades",
x==30~"two of hearts",
x==31~"two of diamonds",
x==32~"two of clubs",
x==33~"three of clubs",
x==34~"three of spades",
x==35~"three of diamonds",
x==36~"three of hearts",
x==37~"seven of hearts",
x==38~"seven of clubs",
x==39~"seven of spades",
x==40~"seven of diamonds",
x==41~"six of diamonds",
x==42~"six of spades",
x==43~"six of clubs",
x==44~"six of hearts",
x==45~"nine of hearts",
x==46~"nine of diamonds",
x==47~"nine of clubs",
x==48~"nine of spades",
x==49~"ten of spades",
x==50~"ten of diamonds",
x==51~"ten of hearts",
x==52~"ten of clubs",
)
}
pred_val <- decode(pred_val)
head(pred_val, 10)## [1] "ace of spades" "ace of clubs" "jack of spades" "ace of clubs"
## [5] "ace of clubs" "ace of hearts" "ace of diamonds" "queen of hearts"
## [9] "seven of hearts" "ace of diamonds"tail(pred_val,10)## [1] "ten of hearts" "jack of clubs" "ten of hearts" "five of hearts"
## [5] "ace of spades" "five of spades" "ace of spades" "ten of clubs"
## [9] "ten of clubs" "nine of spades"levels(as.factor(pred_val))## [1] "ace of clubs" "ace of diamonds" "ace of hearts"
## [4] "ace of spades" "eight of diamonds" "eight of hearts"
## [7] "eight of spades" "five of clubs" "five of diamonds"
## [10] "five of hearts" "five of spades" "four of clubs"
## [13] "four of diamonds" "four of hearts" "four of spades"
## [16] "jack of clubs" "jack of diamonds" "jack of hearts"
## [19] "jack of spades" "joker" "king of clubs"
## [22] "king of diamonds" "king of heart" "king of spades"
## [25] "nine of clubs" "nine of diamonds" "nine of hearts"
## [28] "nine of spades" "queen of clubs" "queen of diamonds"
## [31] "queen of hearts" "queen of spades" "seven of clubs"
## [34] "seven of diamonds" "seven of hearts" "seven of spades"
## [37] "six of clubs" "six of diamonds" "six of hearts"
## [40] "ten of clubs" "ten of diamonds" "ten of hearts"
## [43] "three of clubs" "three of diamonds" "three of hearts"
## [46] "two of hearts" "two of spades"factor(pred_val,levels = levels(as.factor(val_data$class)))## [1] ace of spades ace of clubs jack of spades ace of clubs
## [5] ace of clubs ace of hearts ace of diamonds queen of hearts
## [9] seven of hearts ace of diamonds ace of diamonds ace of diamonds
## [13] ace of spades ace of diamonds ace of hearts ace of spades
## [17] ace of spades ace of spades ace of spades seven of diamonds
## [21] six of diamonds queen of clubs six of hearts six of diamonds
## [25] three of clubs three of diamonds eight of diamonds three of diamonds
## [29] seven of clubs three of diamonds three of diamonds three of diamonds
## [33] king of diamonds ace of spades three of diamonds eight of spades
## [37] eight of spades six of hearts eight of spades three of hearts
## [41] five of clubs three of clubs ace of spades five of clubs
## [45] five of clubs five of diamonds ace of spades five of hearts
## [49] five of hearts three of diamonds five of hearts five of hearts
## [53] three of diamonds five of hearts eight of diamonds eight of spades
## [57] five of clubs five of spades five of spades five of clubs
## [61] four of clubs ace of spades eight of spades four of clubs
## [65] ace of diamonds four of hearts four of diamonds three of diamonds
## [69] five of hearts eight of hearts four of hearts four of hearts
## [73] five of hearts four of hearts eight of hearts four of clubs
## [77] four of spades three of hearts three of hearts four of spades
## [81] jack of clubs jack of clubs jack of spades jack of clubs
## [85] jack of clubs jack of diamonds king of diamonds jack of spades
## [89] jack of spades king of diamonds two of hearts jack of spades
## [93] jack of hearts jack of clubs <NA> jack of clubs
## [97] jack of spades jack of spades jack of clubs jack of spades
## [101] king of diamonds two of spades jack of clubs joker
## [105] jack of clubs jack of clubs king of spades jack of clubs
## [109] jack of clubs jack of clubs king of diamonds king of diamonds
## [113] jack of clubs king of diamonds jack of spades jack of clubs
## [117] two of hearts jack of hearts king of diamonds jack of diamonds
## [121] jack of clubs king of clubs king of spades jack of clubs
## [125] jack of clubs queen of spades three of clubs queen of clubs
## [129] five of clubs six of diamonds queen of diamonds six of clubs
## [133] queen of hearts eight of diamonds six of hearts queen of hearts
## [137] eight of diamonds two of hearts queen of hearts jack of diamonds
## [141] queen of spades queen of spades six of hearts six of diamonds
## [145] queen of clubs king of clubs jack of clubs two of spades
## [149] king of spades jack of clubs jack of clubs king of diamonds
## [153] jack of diamonds king of diamonds king of diamonds king of diamonds
## [157] jack of clubs king of diamonds jack of diamonds jack of clubs
## [161] king of spades jack of clubs jack of clubs jack of clubs
## [165] jack of clubs three of clubs seven of diamonds ace of spades
## [169] two of spades three of clubs three of diamonds three of diamonds
## [173] seven of spades six of clubs five of hearts seven of spades
## [177] three of diamonds three of diamonds eight of hearts three of diamonds
## [181] three of hearts three of hearts seven of hearts jack of spades
## [185] three of hearts seven of hearts queen of clubs seven of diamonds
## [189] queen of clubs seven of hearts seven of spades two of hearts
## [193] seven of spades seven of spades eight of hearts six of clubs
## [197] seven of spades seven of spades two of spades eight of hearts
## [201] seven of diamonds jack of spades seven of diamonds seven of diamonds
## [205] three of hearts six of diamonds ace of spades six of diamonds
## [209] six of diamonds six of diamonds eight of diamonds six of clubs
## [213] jack of spades six of clubs six of clubs six of clubs
## [217] six of clubs six of clubs king of diamonds jack of spades
## [221] queen of spades six of hearts six of hearts six of hearts
## [225] six of diamonds two of hearts five of clubs nine of hearts
## [229] nine of hearts seven of hearts eight of diamonds nine of diamonds
## [233] nine of diamonds ace of diamonds eight of diamonds nine of clubs
## [237] nine of diamonds nine of clubs nine of clubs nine of clubs
## [241] nine of spades nine of spades jack of clubs nine of spades
## [245] nine of spades nine of hearts five of clubs ten of clubs
## [249] five of clubs ten of clubs king of diamonds ten of diamonds
## [253] ace of diamonds ace of hearts four of hearts ten of hearts
## [257] jack of clubs ten of hearts five of hearts ace of spades
## [261] five of spades ace of spades ten of clubs ten of clubs
## [265] nine of spades
## 53 Levels: ace of clubs ace of diamonds ace of hearts ... two of spadesconfusionMatrix(factor(pred_val,levels = levels(as.factor(val_data$class))),
as.factor(val_data$class) )## Confusion Matrix and Statistics
##
## Reference
## Prediction ace of clubs ace of diamonds ace of hearts ace of spades
## ace of clubs 3 0 0 0
## ace of diamonds 0 2 3 0
## ace of hearts 0 1 1 0
## ace of spades 1 0 1 4
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 1 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 1 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 1
## seven of hearts 0 1 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 0 0 0 0
## two of spades 0 0 0 0
## Reference
## Prediction eight of clubs eight of diamonds eight of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 1
## eight of clubs 0 0 0
## eight of diamonds 0 1 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 1
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 1 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 1 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 2 0 0
## six of hearts 1 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 1 0 0
## three of diamonds 0 3 3
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction eight of spades five of clubs five of diamonds
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 1 1
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 3 0 0
## five of clubs 0 3 0
## five of diamonds 0 0 1
## five of hearts 0 0 2
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 1 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 1 0
## three of diamonds 0 0 1
## three of hearts 1 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction five of hearts five of spades four of clubs
## ace of clubs 0 0 0
## ace of diamonds 0 0 1
## ace of hearts 0 0 0
## ace of spades 0 0 1
## eight of clubs 0 0 0
## eight of diamonds 1 0 0
## eight of hearts 0 0 0
## eight of spades 0 1 1
## five of clubs 0 2 0
## five of diamonds 0 0 0
## five of hearts 3 0 0
## five of spades 0 2 0
## four of clubs 0 0 2
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 1 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction four of diamonds four of hearts four of spades
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 1 1 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 1 1 0
## five of spades 0 0 0
## four of clubs 0 0 1
## four of diamonds 1 0 0
## four of hearts 1 3 0
## four of spades 0 0 2
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 1 0 0
## three of hearts 0 0 2
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction jack of clubs jack of diamonds jack of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 4 0 1
## jack of diamonds 0 1 0
## jack of hearts 0 0 1
## jack of spades 1 2 1
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 2 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 1
## two of spades 0 0 0
## Reference
## Prediction jack of spades joker king of clubs king of diamonds
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 2 2 4 1
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 3 0 0 1
## joker 0 1 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 1 0 3
## king of hearts 0 0 0 0
## king of spades 0 0 1 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 0 0 0 0
## two of spades 0 1 0 0
## Reference
## Prediction king of hearts king of spades nine of clubs
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 1
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 1 3 0
## jack of diamonds 1 0 0
## jack of hearts 1 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 1 0
## king of diamonds 1 0 0
## king of hearts 0 0 0
## king of spades 0 1 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 1
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 1
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 1
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 1
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 1 0 0
## two of spades 0 0 0
## Reference
## Prediction nine of diamonds nine of hearts nine of spades
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 1 1 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 1 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 1
## queen of diamonds 1 0 0
## queen of hearts 1 2 0
## queen of spades 0 0 2
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 1 0 0
## six of diamonds 0 0 1
## six of hearts 1 0 1
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 1 0
## two of spades 0 0 0
## Reference
## Prediction queen of clubs queen of diamonds queen of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 2 1 2
## jack of diamonds 0 1 1
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 1 0 0
## king of diamonds 0 3 2
## king of hearts 0 0 0
## king of spades 1 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 1 0 0
## Reference
## Prediction queen of spades seven of clubs seven of diamonds
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 1 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 1
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 4 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 1 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 1 0
## seven of hearts 0 0 0
## seven of spades 0 0 1
## six of clubs 0 0 1
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 2 0
## three of diamonds 0 0 2
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 1 0
## Reference
## Prediction seven of hearts seven of spades six of clubs
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 1 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 1 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 2
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 1
## seven of hearts 0 1 2
## seven of spades 1 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 3 0 0
## three of hearts 0 3 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction six of diamonds six of hearts six of spades ten of clubs
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 1
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 1 1 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 1 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 3 0
## seven of hearts 0 0 0 0
## seven of spades 3 2 0 0
## six of clubs 0 1 0 0
## six of diamonds 0 0 0 4
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 1 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 1 0 0 0
## two of spades 0 1 0 0
## Reference
## Prediction ten of diamonds ten of hearts ten of spades three of clubs
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 1 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 1
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 1 1 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 1 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 2
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 1 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 1
## seven of spades 0 0 0 0
## six of clubs 3 3 0 0
## six of diamonds 0 0 1 0
## six of hearts 0 0 3 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 0 0 0 1
## two of spades 0 0 0 0
## Reference
## Prediction three of diamonds three of hearts three of spades
## ace of clubs 0 0 0
## ace of diamonds 1 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 2 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 1
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 4 0
## nine of diamonds 2 1 0
## nine of hearts 0 0 0
## nine of spades 0 0 4
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction two of clubs two of diamonds two of hearts two of spades
## ace of clubs 0 0 0 0
## ace of diamonds 0 1 0 0
## ace of hearts 0 1 0 0
## ace of spades 0 0 1 1
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 2 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 1 0
## five of spades 0 0 0 1
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 1 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 1 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 1 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 1 0 0 0
## nine of spades 0 0 0 1
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 2 0 0 2
## ten of diamonds 0 1 0 0
## ten of hearts 0 0 2 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 0 0 0 0
## two of spades 0 0 0 0
##
## Overall Statistics
##
## Accuracy : 0.1705
## 95% CI : (0.1271, 0.2214)
## No Information Rate : 0.0189
## P-Value [Acc > NIR] : < 0.00000000000000022
##
## Kappa : 0.1545
##
## Mcnemar's Test P-Value : NA
##
## Statistics by Class:
##
## Class: ace of clubs Class: ace of diamonds
## Sensitivity 0.60000 0.400000
## Specificity 1.00000 0.976834
## Pos Pred Value 1.00000 0.250000
## Neg Pred Value 0.99234 0.988281
## Prevalence 0.01894 0.018939
## Detection Rate 0.01136 0.007576
## Detection Prevalence 0.01136 0.030303
## Balanced Accuracy 0.80000 0.688417
## Class: ace of hearts Class: ace of spades
## Sensitivity 0.200000 0.80000
## Specificity 0.992278 0.96139
## Pos Pred Value 0.333333 0.28571
## Neg Pred Value 0.984674 0.99600
## Prevalence 0.018939 0.01894
## Detection Rate 0.003788 0.01515
## Detection Prevalence 0.011364 0.05303
## Balanced Accuracy 0.596139 0.88069
## Class: eight of clubs Class: eight of diamonds
## Sensitivity 0.00000 0.200000
## Specificity 1.00000 0.976834
## Pos Pred Value NaN 0.142857
## Neg Pred Value 0.98106 0.984436
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.003788
## Detection Prevalence 0.00000 0.026515
## Balanced Accuracy 0.50000 0.588417
## Class: eight of hearts Class: eight of spades
## Sensitivity 0.00000 0.60000
## Specificity 0.98069 0.99228
## Pos Pred Value 0.00000 0.60000
## Neg Pred Value 0.98069 0.99228
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.01136
## Detection Prevalence 0.01894 0.01894
## Balanced Accuracy 0.49035 0.79614
## Class: five of clubs Class: five of diamonds
## Sensitivity 0.60000 0.200000
## Specificity 0.97683 1.000000
## Pos Pred Value 0.33333 1.000000
## Neg Pred Value 0.99216 0.984791
## Prevalence 0.01894 0.018939
## Detection Rate 0.01136 0.003788
## Detection Prevalence 0.03409 0.003788
## Balanced Accuracy 0.78842 0.600000
## Class: five of hearts Class: five of spades
## Sensitivity 0.60000 0.400000
## Specificity 0.97683 0.996139
## Pos Pred Value 0.33333 0.666667
## Neg Pred Value 0.99216 0.988506
## Prevalence 0.01894 0.018939
## Detection Rate 0.01136 0.007576
## Detection Prevalence 0.03409 0.011364
## Balanced Accuracy 0.78842 0.698069
## Class: four of clubs Class: four of diamonds
## Sensitivity 0.400000 0.200000
## Specificity 0.996139 1.000000
## Pos Pred Value 0.666667 1.000000
## Neg Pred Value 0.988506 0.984791
## Prevalence 0.018939 0.018939
## Detection Rate 0.007576 0.003788
## Detection Prevalence 0.011364 0.003788
## Balanced Accuracy 0.698069 0.600000
## Class: four of hearts Class: four of spades
## Sensitivity 0.60000 0.400000
## Specificity 0.99228 1.000000
## Pos Pred Value 0.60000 1.000000
## Neg Pred Value 0.99228 0.988550
## Prevalence 0.01894 0.018939
## Detection Rate 0.01136 0.007576
## Detection Prevalence 0.01894 0.007576
## Balanced Accuracy 0.79614 0.700000
## Class: jack of clubs Class: jack of diamonds
## Sensitivity 0.80000 0.200000
## Specificity 0.90347 0.984556
## Pos Pred Value 0.13793 0.200000
## Neg Pred Value 0.99574 0.984556
## Prevalence 0.01894 0.018939
## Detection Rate 0.01515 0.003788
## Detection Prevalence 0.10985 0.018939
## Balanced Accuracy 0.85174 0.592278
## Class: jack of hearts Class: jack of spades Class: joker
## Sensitivity 0.250000 0.60000 0.200000
## Specificity 0.996154 0.96139 1.000000
## Pos Pred Value 0.500000 0.23077 1.000000
## Neg Pred Value 0.988550 0.99203 0.984791
## Prevalence 0.015152 0.01894 0.018939
## Detection Rate 0.003788 0.01136 0.003788
## Detection Prevalence 0.007576 0.04924 0.003788
## Balanced Accuracy 0.623077 0.78069 0.600000
## Class: king of clubs Class: king of diamonds
## Sensitivity 0.000000 0.60000
## Specificity 0.992278 0.95367
## Pos Pred Value 0.000000 0.20000
## Neg Pred Value 0.980916 0.99197
## Prevalence 0.018939 0.01894
## Detection Rate 0.000000 0.01136
## Detection Prevalence 0.007576 0.05682
## Balanced Accuracy 0.496139 0.77683
## Class: king of hearts Class: king of spades
## Sensitivity 0.00000 0.200000
## Specificity 1.00000 0.988417
## Pos Pred Value NaN 0.250000
## Neg Pred Value 0.98106 0.984615
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.003788
## Detection Prevalence 0.00000 0.015152
## Balanced Accuracy 0.50000 0.594208
## Class: nine of clubs Class: nine of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.98842
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98084
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.01136
## Balanced Accuracy 0.49228 0.49421
## Class: nine of hearts Class: nine of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98842 0.98069
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98084 0.98069
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01136 0.01894
## Balanced Accuracy 0.49421 0.49035
## Class: queen of clubs Class: queen of diamonds
## Sensitivity 0.00000 0.000000
## Specificity 0.98069 0.996139
## Pos Pred Value 0.00000 0.000000
## Neg Pred Value 0.98069 0.980989
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.000000
## Detection Prevalence 0.01894 0.003788
## Balanced Accuracy 0.49035 0.498069
## Class: queen of hearts Class: queen of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.98456
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98077
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.01515
## Balanced Accuracy 0.49228 0.49228
## Class: seven of clubs Class: seven of diamonds
## Sensitivity 0.000000 0.00000
## Specificity 0.996139 0.97683
## Pos Pred Value 0.000000 0.00000
## Neg Pred Value 0.980989 0.98062
## Prevalence 0.018939 0.01894
## Detection Rate 0.000000 0.00000
## Detection Prevalence 0.003788 0.02273
## Balanced Accuracy 0.498069 0.48842
## Class: seven of hearts Class: seven of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98069 0.97297
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98069 0.98054
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01894 0.02652
## Balanced Accuracy 0.49035 0.48649
## Class: six of clubs Class: six of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.96525 0.96525
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98039 0.98039
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.03409 0.03409
## Balanced Accuracy 0.48263 0.48263
## Class: six of hearts Class: six of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.97297 1.00000
## Pos Pred Value 0.00000 NaN
## Neg Pred Value 0.98054 0.98106
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.02652 0.00000
## Balanced Accuracy 0.48649 0.50000
## Class: ten of clubs Class: ten of diamonds
## Sensitivity 0.00000 0.000000
## Specificity 0.98456 0.996139
## Pos Pred Value 0.00000 0.000000
## Neg Pred Value 0.98077 0.980989
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.000000
## Detection Prevalence 0.01515 0.003788
## Balanced Accuracy 0.49228 0.498069
## Class: ten of hearts Class: ten of spades
## Sensitivity 0.000000 0.00000
## Specificity 0.992278 1.00000
## Pos Pred Value 0.000000 NaN
## Neg Pred Value 0.980916 0.98106
## Prevalence 0.018939 0.01894
## Detection Rate 0.000000 0.00000
## Detection Prevalence 0.007576 0.00000
## Balanced Accuracy 0.496139 0.50000
## Class: three of clubs Class: three of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98069 0.94595
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98069 0.98000
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01894 0.05303
## Balanced Accuracy 0.49035 0.47297
## Class: three of hearts Class: three of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.97297 1.00000
## Pos Pred Value 0.00000 NaN
## Neg Pred Value 0.98054 0.98106
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.02652 0.00000
## Balanced Accuracy 0.48649 0.50000
## Class: two of clubs Class: two of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 1.00000 1.00000
## Pos Pred Value NaN NaN
## Neg Pred Value 0.98106 0.98106
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.00000 0.00000
## Balanced Accuracy 0.50000 0.50000
## Class: two of hearts Class: two of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98069 0.98456
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98069 0.98077
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01894 0.01515
## Balanced Accuracy 0.49035 0.49228# Set Initial Random Weight
tensorflow::tf$random$set_seed(123)
model_tunning <- keras_model_sequential(name = "CNN_Model_Tunningl") %>%
# Convolution Layer 1
layer_conv_2d(filters = 64,
kernel_size = c(3,3),
padding = "same",
activation = "relu",
input_shape = c(target_size, 3)
) %>%
# Max Pooling Layer 1
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 2
layer_conv_2d(filters = 64,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 2
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 3
layer_conv_2d(filters = 128,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 3
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 4
layer_conv_2d(filters = 128,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 4
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Convolution Layer 5
layer_conv_2d(filters = 128,
kernel_size = c(3,3),
padding = "same",
activation = "relu"
) %>%
# Max Pooling Layer 5
layer_max_pooling_2d(pool_size = c(2,2),
strides = c(2,2)) %>%
# Flattening Layer
layer_flatten() %>%
# Dense Layer1
layer_dense(units = 128,
activation = "relu") %>%
# Output Layer
layer_dense(units = output_n,
activation = "softmax",
name = "Output")
# Compile Model
model_tunning %>%
compile(
loss = "categorical_crossentropy",
optimizer = optimizer_adam(learning_rate = 0.001),
metrics = "accuracy"
)
model_tunning## Model: "CNN_Model_Tunningl"
## ________________________________________________________________________________
## Layer (type) Output Shape Param #
## ================================================================================
## conv2d_7 (Conv2D) (None, 64, 64, 64) 1792
## max_pooling2d_7 (MaxPooling2D) (None, 32, 32, 64) 0
## conv2d_6 (Conv2D) (None, 32, 32, 64) 36928
## max_pooling2d_6 (MaxPooling2D) (None, 16, 16, 64) 0
## conv2d_5 (Conv2D) (None, 16, 16, 128) 73856
## max_pooling2d_5 (MaxPooling2D) (None, 8, 8, 128) 0
## conv2d_4 (Conv2D) (None, 8, 8, 128) 147584
## max_pooling2d_4 (MaxPooling2D) (None, 4, 4, 128) 0
## conv2d_3 (Conv2D) (None, 4, 4, 128) 147584
## max_pooling2d_3 (MaxPooling2D) (None, 2, 2, 128) 0
## flatten_1 (Flatten) (None, 512) 0
## dense_1 (Dense) (None, 128) 65664
## Output (Dense) (None, 53) 6837
## ================================================================================
## Total params: 480,245
## Trainable params: 480,245
## Non-trainable params: 0
## ________________________________________________________________________________# Fit data into model
history1 <- model_tunning %>%
fit(
# training data
train_image_array_gen,
# training epochs
steps_per_epoch = as.integer(train_samples / batch_size),
epochs = 25,
# testing data
validation_data = test_image_array_gen,
validation_steps = as.integer(test_samples / batch_size)
)## Epoch 1/25
## 29/29 - 53s - loss: 3.9588 - accuracy: 0.0200 - val_loss: 3.8960 - val_accuracy: 0.0234 - 53s/epoch - 2s/step
## Epoch 2/25
## 29/29 - 47s - loss: 3.6972 - accuracy: 0.0487 - val_loss: 3.6608 - val_accuracy: 0.0586 - 47s/epoch - 2s/step
## Epoch 3/25
## 29/29 - 47s - loss: 3.2035 - accuracy: 0.1193 - val_loss: 3.2565 - val_accuracy: 0.0938 - 47s/epoch - 2s/step
## Epoch 4/25
## 29/29 - 47s - loss: 2.9463 - accuracy: 0.1540 - val_loss: 2.9088 - val_accuracy: 0.1562 - 47s/epoch - 2s/step
## Epoch 5/25
## 29/29 - 55s - loss: 2.7243 - accuracy: 0.1994 - val_loss: 2.6108 - val_accuracy: 0.2305 - 55s/epoch - 2s/step
## Epoch 6/25
## 29/29 - 50s - loss: 2.5878 - accuracy: 0.2330 - val_loss: 2.7517 - val_accuracy: 0.1992 - 50s/epoch - 2s/step
## Epoch 7/25
## 29/29 - 47s - loss: 2.4663 - accuracy: 0.2605 - val_loss: 2.4135 - val_accuracy: 0.2773 - 47s/epoch - 2s/step
## Epoch 8/25
## 29/29 - 48s - loss: 2.3653 - accuracy: 0.2937 - val_loss: 2.3360 - val_accuracy: 0.3086 - 48s/epoch - 2s/step
## Epoch 9/25
## 29/29 - 46s - loss: 2.2986 - accuracy: 0.3214 - val_loss: 2.2072 - val_accuracy: 0.3086 - 46s/epoch - 2s/step
## Epoch 10/25
## 29/29 - 51s - loss: 2.2029 - accuracy: 0.3533 - val_loss: 2.2420 - val_accuracy: 0.3555 - 51s/epoch - 2s/step
## Epoch 11/25
## 29/29 - 60s - loss: 2.1498 - accuracy: 0.3667 - val_loss: 2.0724 - val_accuracy: 0.3984 - 60s/epoch - 2s/step
## Epoch 12/25
## 29/29 - 55s - loss: 2.0792 - accuracy: 0.3948 - val_loss: 1.9790 - val_accuracy: 0.3945 - 55s/epoch - 2s/step
## Epoch 13/25
## 29/29 - 74s - loss: 2.0018 - accuracy: 0.4236 - val_loss: 1.8587 - val_accuracy: 0.4648 - 74s/epoch - 3s/step
## Epoch 14/25
## 29/29 - 74s - loss: 1.9321 - accuracy: 0.4503 - val_loss: 1.9181 - val_accuracy: 0.5039 - 74s/epoch - 3s/step
## Epoch 15/25
## 29/29 - 75s - loss: 1.8619 - accuracy: 0.4746 - val_loss: 1.8055 - val_accuracy: 0.4727 - 75s/epoch - 3s/step
## Epoch 16/25
## 29/29 - 64s - loss: 1.8034 - accuracy: 0.4910 - val_loss: 1.7217 - val_accuracy: 0.5039 - 64s/epoch - 2s/step
## Epoch 17/25
## 29/29 - 61s - loss: 1.7312 - accuracy: 0.5109 - val_loss: 1.5703 - val_accuracy: 0.5703 - 61s/epoch - 2s/step
## Epoch 18/25
## 29/29 - 61s - loss: 1.6748 - accuracy: 0.5309 - val_loss: 1.6233 - val_accuracy: 0.5625 - 61s/epoch - 2s/step
## Epoch 19/25
## 29/29 - 60s - loss: 1.6384 - accuracy: 0.5419 - val_loss: 1.5639 - val_accuracy: 0.5508 - 60s/epoch - 2s/step
## Epoch 20/25
## 29/29 - 59s - loss: 1.5860 - accuracy: 0.5575 - val_loss: 1.4903 - val_accuracy: 0.5742 - 59s/epoch - 2s/step
## Epoch 21/25
## 29/29 - 62s - loss: 1.5755 - accuracy: 0.5666 - val_loss: 1.5193 - val_accuracy: 0.5664 - 62s/epoch - 2s/step
## Epoch 22/25
## 29/29 - 53s - loss: 1.5442 - accuracy: 0.5676 - val_loss: 1.4551 - val_accuracy: 0.5664 - 53s/epoch - 2s/step
## Epoch 23/25
## 29/29 - 52s - loss: 1.5007 - accuracy: 0.5854 - val_loss: 1.4331 - val_accuracy: 0.5742 - 52s/epoch - 2s/step
## Epoch 24/25
## 29/29 - 48s - loss: 1.4713 - accuracy: 0.5859 - val_loss: 1.4432 - val_accuracy: 0.5859 - 48s/epoch - 2s/step
## Epoch 25/25
## 29/29 - 55s - loss: 1.4569 - accuracy: 0.5958 - val_loss: 1.4632 - val_accuracy: 0.5898 - 55s/epoch - 2s/stepplot(history1)
Insight: Hasil dari model tuning yang diperlakukan kepada data test
diperoleh: accuracy = 0.5581, loss = 1.45 val accuracy = 0.6289, val
loss = 1.44
dengan epoch sebanyak 25x diperoleh hasil yang lumayan memuaskan. hasil dapat di optimasi lagi dengan menambahkan jumlah epoch, akan tetapi beban komputasi akan bertambah secara signifikan. sehingga dapat disimpulkan hasil menggunakan model tuning yang telah dibuat berhasil naik signifikan dibanding model yang tidak dilakukan tuning.
pred_val_tun <- predict(model_tunning, val_x) %>% k_argmax() %>% as.array()## 9/9 - 1s - 525ms/epoch - 58ms/stephead(pred_val_tun, 10)## [1] 3 0 30 0 3 1 1 46 2 1pred_val_tun <- sapply(pred_val_tun, decode)
head(pred_val_tun, 10)## [1] "ace of spades" "ace of clubs" "two of hearts" "ace of clubs"
## [5] "ace of spades" "ace of diamonds" "ace of diamonds" "nine of diamonds"
## [9] "ace of hearts" "ace of diamonds"confusionMatrix(factor(pred_val_tun,levels = levels(as.factor(val_data$class))),
as.factor(val_data$class) )## Confusion Matrix and Statistics
##
## Reference
## Prediction ace of clubs ace of diamonds ace of hearts ace of spades
## ace of clubs 2 0 0 0
## ace of diamonds 0 3 0 0
## ace of hearts 0 1 4 0
## ace of spades 2 0 1 4
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 1 0 0
## nine of hearts 0 0 0 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 1
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 1 0 0 0
## two of spades 0 0 0 0
## Reference
## Prediction eight of clubs eight of diamonds eight of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 1
## eight of clubs 5 0 0
## eight of diamonds 0 5 0
## eight of hearts 0 0 3
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 1
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction eight of spades five of clubs five of diamonds
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 5 0 0
## five of clubs 0 4 0
## five of diamonds 0 0 4
## five of hearts 0 0 0
## five of spades 0 1 1
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction five of hearts five of spades four of clubs
## ace of clubs 0 0 1
## ace of diamonds 0 0 1
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 4 0 0
## five of spades 0 5 0
## four of clubs 0 0 2
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 1
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 1 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction four of diamonds four of hearts four of spades
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 4 0 0
## four of hearts 0 4 0
## four of spades 0 0 5
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 1 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 1 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction jack of clubs jack of diamonds jack of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 2 0 0
## jack of diamonds 0 1 1
## jack of hearts 0 0 1
## jack of spades 0 1 0
## joker 0 0 0
## king of clubs 2 0 0
## king of diamonds 0 2 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 1 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 1 1
## two of diamonds 0 0 2
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction jack of spades joker king of clubs king of diamonds
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 1
## jack of hearts 0 0 0 0
## jack of spades 3 0 0 0
## joker 0 3 0 0
## king of clubs 0 0 2 0
## king of diamonds 0 0 0 3
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 2 0 2 0
## two of diamonds 0 0 0 0
## two of hearts 0 1 0 1
## two of spades 0 1 0 0
## Reference
## Prediction king of hearts king of spades nine of clubs
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 1
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 2 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 2 1 0
## king of hearts 0 0 0
## king of spades 0 2 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 4
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 2 0
## two of diamonds 0 0 0
## two of hearts 1 0 0
## two of spades 0 0 0
## Reference
## Prediction nine of diamonds nine of hearts nine of spades
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 4 0 0
## queen of hearts 0 5 0
## queen of spades 0 0 2
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 3
## six of spades 1 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction queen of clubs queen of diamonds queen of hearts
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 1 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 1 5
## two of hearts 0 3 0
## two of spades 5 0 0
## Reference
## Prediction queen of spades seven of clubs seven of diamonds
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 1 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 1 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 1 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 2 0
## three of diamonds 0 0 0
## three of hearts 0 1 0
## three of spades 0 0 5
## two of clubs 4 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction seven of hearts seven of spades six of clubs
## ace of clubs 0 0 0
## ace of diamonds 0 0 0
## ace of hearts 0 0 0
## ace of spades 0 1 0
## eight of clubs 0 0 2
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 0 0
## nine of diamonds 0 0 0
## nine of hearts 0 0 0
## nine of spades 0 0 0
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 3
## seven of spades 0 0 0
## six of clubs 1 0 0
## six of diamonds 0 0 0
## six of hearts 0 1 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 0
## three of clubs 0 0 0
## three of diamonds 4 0 0
## three of hearts 0 3 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction six of diamonds six of hearts six of spades ten of clubs
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 1
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 1 0
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 4 0 0 0
## seven of diamonds 0 1 4 0
## seven of hearts 0 0 0 0
## seven of spades 0 4 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 3
## six of hearts 0 0 0 1
## six of spades 0 0 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 0 0 0
## two of hearts 1 0 0 0
## two of spades 0 0 0 0
## Reference
## Prediction ten of diamonds ten of hearts ten of spades three of clubs
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 0 0 0 0
## eight of hearts 0 0 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 0 0
## nine of diamonds 0 0 0 0
## nine of hearts 0 0 0 5
## nine of spades 0 0 0 0
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 2 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 5 0
## six of spades 4 1 0 0
## ten of clubs 0 0 0 0
## ten of diamonds 0 0 0 0
## ten of hearts 0 0 0 0
## ten of spades 0 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 0
## two of diamonds 0 1 0 0
## two of hearts 1 0 0 0
## two of spades 0 1 0 0
## Reference
## Prediction three of diamonds three of hearts three of spades
## ace of clubs 0 0 0
## ace of diamonds 1 0 0
## ace of hearts 0 0 0
## ace of spades 0 0 0
## eight of clubs 0 0 0
## eight of diamonds 0 0 0
## eight of hearts 0 0 0
## eight of spades 0 0 0
## five of clubs 0 0 0
## five of diamonds 0 0 0
## five of hearts 0 0 0
## five of spades 0 0 0
## four of clubs 0 0 0
## four of diamonds 0 0 0
## four of hearts 0 0 0
## four of spades 0 0 0
## jack of clubs 0 0 0
## jack of diamonds 0 0 0
## jack of hearts 0 0 0
## jack of spades 0 0 0
## joker 0 0 0
## king of clubs 0 0 0
## king of diamonds 0 0 0
## king of hearts 0 0 0
## king of spades 0 0 0
## nine of clubs 0 4 0
## nine of diamonds 4 1 0
## nine of hearts 0 0 0
## nine of spades 0 0 4
## queen of clubs 0 0 0
## queen of diamonds 0 0 0
## queen of hearts 0 0 0
## queen of spades 0 0 0
## seven of clubs 0 0 0
## seven of diamonds 0 0 0
## seven of hearts 0 0 0
## seven of spades 0 0 0
## six of clubs 0 0 0
## six of diamonds 0 0 0
## six of hearts 0 0 0
## six of spades 0 0 0
## ten of clubs 0 0 0
## ten of diamonds 0 0 0
## ten of hearts 0 0 0
## ten of spades 0 0 1
## three of clubs 0 0 0
## three of diamonds 0 0 0
## three of hearts 0 0 0
## three of spades 0 0 0
## two of clubs 0 0 0
## two of diamonds 0 0 0
## two of hearts 0 0 0
## two of spades 0 0 0
## Reference
## Prediction two of clubs two of diamonds two of hearts two of spades
## ace of clubs 0 0 0 0
## ace of diamonds 0 0 0 0
## ace of hearts 0 0 0 0
## ace of spades 0 0 0 0
## eight of clubs 0 0 0 0
## eight of diamonds 0 1 0 0
## eight of hearts 0 1 0 0
## eight of spades 0 0 0 0
## five of clubs 0 0 0 0
## five of diamonds 0 0 0 0
## five of hearts 0 0 0 0
## five of spades 0 0 0 0
## four of clubs 0 0 0 0
## four of diamonds 0 0 0 0
## four of hearts 0 0 0 0
## four of spades 0 0 0 0
## jack of clubs 0 0 0 0
## jack of diamonds 0 0 0 0
## jack of hearts 0 0 0 0
## jack of spades 0 0 0 0
## joker 0 0 0 0
## king of clubs 0 0 0 0
## king of diamonds 0 0 0 0
## king of hearts 0 0 0 0
## king of spades 0 0 0 0
## nine of clubs 0 0 1 0
## nine of diamonds 0 2 0 0
## nine of hearts 1 0 0 0
## nine of spades 0 0 0 1
## queen of clubs 0 0 0 0
## queen of diamonds 0 0 0 0
## queen of hearts 0 0 0 0
## queen of spades 0 0 0 0
## seven of clubs 0 0 0 0
## seven of diamonds 0 0 0 0
## seven of hearts 0 0 0 0
## seven of spades 0 0 0 0
## six of clubs 0 0 0 0
## six of diamonds 0 0 0 0
## six of hearts 0 0 0 0
## six of spades 0 0 0 0
## ten of clubs 0 0 0 3
## ten of diamonds 0 1 0 0
## ten of hearts 0 0 2 0
## ten of spades 4 0 0 0
## three of clubs 0 0 0 0
## three of diamonds 0 0 0 0
## three of hearts 0 0 0 0
## three of spades 0 0 0 0
## two of clubs 0 0 0 1
## two of diamonds 0 0 0 0
## two of hearts 0 0 2 0
## two of spades 0 0 0 0
##
## Overall Statistics
##
## Accuracy : 0.3106
## 95% CI : (0.2553, 0.3702)
## No Information Rate : 0.0189
## P-Value [Acc > NIR] : < 0.00000000000000022
##
## Kappa : 0.2973
##
## Mcnemar's Test P-Value : NA
##
## Statistics by Class:
##
## Class: ace of clubs Class: ace of diamonds
## Sensitivity 0.400000 0.60000
## Specificity 0.996139 0.99228
## Pos Pred Value 0.666667 0.60000
## Neg Pred Value 0.988506 0.99228
## Prevalence 0.018939 0.01894
## Detection Rate 0.007576 0.01136
## Detection Prevalence 0.011364 0.01894
## Balanced Accuracy 0.698069 0.79614
## Class: ace of hearts Class: ace of spades
## Sensitivity 0.80000 0.80000
## Specificity 0.99614 0.98069
## Pos Pred Value 0.80000 0.44444
## Neg Pred Value 0.99614 0.99608
## Prevalence 0.01894 0.01894
## Detection Rate 0.01515 0.01515
## Detection Prevalence 0.01894 0.03409
## Balanced Accuracy 0.89807 0.89035
## Class: eight of clubs Class: eight of diamonds
## Sensitivity 1.00000 1.00000
## Specificity 0.98456 0.99614
## Pos Pred Value 0.55556 0.83333
## Neg Pred Value 1.00000 1.00000
## Prevalence 0.01894 0.01894
## Detection Rate 0.01894 0.01894
## Detection Prevalence 0.03409 0.02273
## Balanced Accuracy 0.99228 0.99807
## Class: eight of hearts Class: eight of spades
## Sensitivity 0.60000 1.00000
## Specificity 0.99614 1.00000
## Pos Pred Value 0.75000 1.00000
## Neg Pred Value 0.99231 1.00000
## Prevalence 0.01894 0.01894
## Detection Rate 0.01136 0.01894
## Detection Prevalence 0.01515 0.01894
## Balanced Accuracy 0.79807 1.00000
## Class: five of clubs Class: five of diamonds
## Sensitivity 0.80000 0.80000
## Specificity 0.99614 1.00000
## Pos Pred Value 0.80000 1.00000
## Neg Pred Value 0.99614 0.99615
## Prevalence 0.01894 0.01894
## Detection Rate 0.01515 0.01515
## Detection Prevalence 0.01894 0.01515
## Balanced Accuracy 0.89807 0.90000
## Class: five of hearts Class: five of spades
## Sensitivity 0.80000 1.00000
## Specificity 1.00000 0.99228
## Pos Pred Value 1.00000 0.71429
## Neg Pred Value 0.99615 1.00000
## Prevalence 0.01894 0.01894
## Detection Rate 0.01515 0.01894
## Detection Prevalence 0.01515 0.02652
## Balanced Accuracy 0.90000 0.99614
## Class: four of clubs Class: four of diamonds
## Sensitivity 0.400000 0.80000
## Specificity 0.996139 1.00000
## Pos Pred Value 0.666667 1.00000
## Neg Pred Value 0.988506 0.99615
## Prevalence 0.018939 0.01894
## Detection Rate 0.007576 0.01515
## Detection Prevalence 0.011364 0.01515
## Balanced Accuracy 0.698069 0.90000
## Class: four of hearts Class: four of spades
## Sensitivity 0.80000 1.00000
## Specificity 1.00000 1.00000
## Pos Pred Value 1.00000 1.00000
## Neg Pred Value 0.99615 1.00000
## Prevalence 0.01894 0.01894
## Detection Rate 0.01515 0.01894
## Detection Prevalence 0.01515 0.01894
## Balanced Accuracy 0.90000 1.00000
## Class: jack of clubs Class: jack of diamonds
## Sensitivity 0.400000 0.200000
## Specificity 1.000000 0.992278
## Pos Pred Value 1.000000 0.333333
## Neg Pred Value 0.988550 0.984674
## Prevalence 0.018939 0.018939
## Detection Rate 0.007576 0.003788
## Detection Prevalence 0.007576 0.011364
## Balanced Accuracy 0.700000 0.596139
## Class: jack of hearts Class: jack of spades Class: joker
## Sensitivity 0.200000 0.60000 0.60000
## Specificity 0.992278 0.99228 1.00000
## Pos Pred Value 0.333333 0.60000 1.00000
## Neg Pred Value 0.984674 0.99228 0.99234
## Prevalence 0.018939 0.01894 0.01894
## Detection Rate 0.003788 0.01136 0.01136
## Detection Prevalence 0.011364 0.01894 0.01136
## Balanced Accuracy 0.596139 0.79614 0.80000
## Class: king of clubs Class: king of diamonds
## Sensitivity 0.500000 0.60000
## Specificity 0.992308 0.97683
## Pos Pred Value 0.500000 0.33333
## Neg Pred Value 0.992308 0.99216
## Prevalence 0.015152 0.01894
## Detection Rate 0.007576 0.01136
## Detection Prevalence 0.015152 0.03409
## Balanced Accuracy 0.746154 0.78842
## Class: king of hearts Class: king of spades
## Sensitivity 0.00000 0.400000
## Specificity 1.00000 1.000000
## Pos Pred Value NaN 1.000000
## Neg Pred Value 0.98106 0.988550
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.007576
## Detection Prevalence 0.00000 0.007576
## Balanced Accuracy 0.50000 0.700000
## Class: nine of clubs Class: nine of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98069 0.96911
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98069 0.98047
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01894 0.03030
## Balanced Accuracy 0.49035 0.48456
## Class: nine of hearts Class: nine of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.97297 0.98069
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98054 0.98069
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.02652 0.01894
## Balanced Accuracy 0.48649 0.49035
## Class: queen of clubs Class: queen of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.98456
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98077
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.01515
## Balanced Accuracy 0.49228 0.49228
## Class: queen of hearts Class: queen of spades
## Sensitivity 0.00000 0.000000
## Specificity 0.98069 0.992278
## Pos Pred Value 0.00000 0.000000
## Neg Pred Value 0.98069 0.980916
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.000000
## Detection Prevalence 0.01894 0.007576
## Balanced Accuracy 0.49035 0.496139
## Class: seven of clubs Class: seven of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.98069
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98069
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.01894
## Balanced Accuracy 0.49228 0.49035
## Class: seven of hearts Class: seven of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.98069
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98069
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.01894
## Balanced Accuracy 0.49228 0.49035
## Class: six of clubs Class: six of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.98842 0.98456
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98084 0.98077
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01136 0.01515
## Balanced Accuracy 0.49421 0.49228
## Class: six of hearts Class: six of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.95753 0.97683
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98024 0.98062
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.04167 0.02273
## Balanced Accuracy 0.47876 0.48842
## Class: ten of clubs Class: ten of diamonds
## Sensitivity 0.00000 0.000000
## Specificity 0.98842 0.996139
## Pos Pred Value 0.00000 0.000000
## Neg Pred Value 0.98084 0.980989
## Prevalence 0.01894 0.018939
## Detection Rate 0.00000 0.000000
## Detection Prevalence 0.01136 0.003788
## Balanced Accuracy 0.49421 0.498069
## Class: ten of hearts Class: ten of spades
## Sensitivity 0.000000 0.00000
## Specificity 0.992278 0.98069
## Pos Pred Value 0.000000 0.00000
## Neg Pred Value 0.980916 0.98069
## Prevalence 0.018939 0.01894
## Detection Rate 0.000000 0.00000
## Detection Prevalence 0.007576 0.01894
## Balanced Accuracy 0.496139 0.49035
## Class: three of clubs Class: three of diamonds
## Sensitivity 0.000000 0.00000
## Specificity 0.992278 0.98069
## Pos Pred Value 0.000000 0.00000
## Neg Pred Value 0.980916 0.98069
## Prevalence 0.018939 0.01894
## Detection Rate 0.000000 0.00000
## Detection Prevalence 0.007576 0.01894
## Balanced Accuracy 0.496139 0.49035
## Class: three of hearts Class: three of spades
## Sensitivity 0.00000 0.00000
## Specificity 0.98456 0.97683
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98077 0.98062
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.01515 0.02273
## Balanced Accuracy 0.49228 0.48842
## Class: two of clubs Class: two of diamonds
## Sensitivity 0.00000 0.00000
## Specificity 0.94981 0.96139
## Pos Pred Value 0.00000 0.00000
## Neg Pred Value 0.98008 0.98031
## Prevalence 0.01894 0.01894
## Detection Rate 0.00000 0.00000
## Detection Prevalence 0.04924 0.03788
## Balanced Accuracy 0.47490 0.48069
## Class: two of hearts Class: two of spades
## Sensitivity 0.400000 0.00000
## Specificity 0.965251 0.97297
## Pos Pred Value 0.181818 0.00000
## Neg Pred Value 0.988142 0.98054
## Prevalence 0.018939 0.01894
## Detection Rate 0.007576 0.00000
## Detection Prevalence 0.041667 0.02652
## Balanced Accuracy 0.682625 0.48649