2.1 How many pixel?

sqrt(784) # ukuran gambar

TRUE [1] 28

2.2 Check Proportion of Target (Y)

plot(table(mnist$label))

9.1 Build Base Model

model <- keras_model_sequential()

9.2 Make Neural-net Architecture

model %>% 
  layer_dense(units = 16*2, activation = "relu", input_shape = ncol(train_x_keras), name = "hidden1") %>% 
  layer_dense(units = 16, activation = "relu", name = "hidden2") %>% 
  layer_dense(units = 10, activation = "softmax", name = "output")

model %>% 
  compile(loss = "categorical_crossentropy", 
          optimizer = optimizer_adam(lr = 0.001), 
          metrics = c("accuracy"))

summary(model)

TRUE Model: "sequential"
TRUE ___________________________________________________________________________
TRUE Layer (type)                     Output Shape                  Param #     
TRUE ===========================================================================
TRUE hidden1 (Dense)                  (None, 32)                    25120       
TRUE ___________________________________________________________________________
TRUE hidden2 (Dense)                  (None, 16)                    528         
TRUE ___________________________________________________________________________
TRUE output (Dense)                   (None, 10)                    170         
TRUE ===========================================================================
TRUE Total params: 25,818
TRUE Trainable params: 25,818
TRUE Non-trainable params: 0
TRUE ___________________________________________________________________________

9.3 fit model

#model %>% 
 # fit(train_x_keras, train_y_keras, epoch = 15, batch_size = 10)

Create Model

The core data structure of Keras is a model, a way to organize layers. The simplest type of model is the Sequential model, a linear stack of layers.

The input_shape argument to the first layer specifies the shape of the input data (a length 784 numeric vector representing a grayscale image). The final layer outputs a length 10 numeric vector (probabilities for each digit) using a softmax activation function.

source: https://keras.rstudio.com/

model_2 <- keras_model_sequential()

model_2 %>% 
  layer_dense(units = 256, activation = 'relu', input_shape = c(784)) %>% 
  layer_dropout(rate = 0.4) %>% 
  layer_dense(units = 128, activation = 'relu') %>%
  layer_dropout(rate = 0.3) %>%
  layer_dense(units = 10, activation = 'softmax')

#compile 
model_2 %>% compile(
  loss = "categorical_crossentropy",
  optimizer = optimizer_rmsprop(),
  metrics = "accuracy"
)

# model_2 %>% fit(
#  train_x_keras, train_y_keras, 
#  epochs = 30, batch_size = 128, 
#  validation_split = 0.2
#)
 
summary(model_2)

TRUE Model: "sequential_1"
TRUE ___________________________________________________________________________
TRUE Layer (type)                     Output Shape                  Param #     
TRUE ===========================================================================
TRUE dense (Dense)                    (None, 256)                   200960      
TRUE ___________________________________________________________________________
TRUE dropout (Dropout)                (None, 256)                   0           
TRUE ___________________________________________________________________________
TRUE dense_1 (Dense)                  (None, 128)                   32896       
TRUE ___________________________________________________________________________
TRUE dropout_1 (Dropout)              (None, 128)                   0           
TRUE ___________________________________________________________________________
TRUE dense_2 (Dense)                  (None, 10)                    1290        
TRUE ===========================================================================
TRUE Total params: 235,146
TRUE Trainable params: 235,146
TRUE Non-trainable params: 0
TRUE ___________________________________________________________________________

Evaluate Model 1

The Accuracy of Model 1 is 87.78

#model_2 %>% save_model_hdf5("model_2.hdf5")

model_2 <- load_model_hdf5("model_2.hdf5")

pred <- keras::predict_classes(object = model_2, x = test_x_keras)

categories <- c("T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot")

pred <- factor(pred, labels=categories)

test_y <- factor (test_y, labels = categories)

caret::confusionMatrix(as.factor(pred), as.factor(test_y))

TRUE Confusion Matrix and Statistics
TRUE 
TRUE           Reference
TRUE Prediction T-shirt Trouser Pullover Dress Coat Sandal Shirt Sneaker  Bag
TRUE   T-shirt     1021       1       17    23    1      0   117       0    9
TRUE   Trouser        2    1133        0     4    0      0     1       0    0
TRUE   Pullover       7       4      857     5   66      0    41       0    4
TRUE   Dress         32      18        9  1056   31      0    23       0    7
TRUE   Coat           4       3      101    58  960      0    47       0    4
TRUE   Sandal         0       0        0     0    0   1175     0      13    6
TRUE   Shirt        155       4      225    33  151      0   928       0   17
TRUE           Reference
TRUE Prediction Boot
TRUE   T-shirt     0
TRUE   Trouser     0
TRUE   Pullover    0
TRUE   Dress       0
TRUE   Coat        0
TRUE   Sandal      9
TRUE   Shirt       0
TRUE  [ reached getOption("max.print") -- omitted 3 rows ]
TRUE 
TRUE Overall Statistics
TRUE                                                
TRUE                Accuracy : 0.8832               
TRUE                  95% CI : (0.8773, 0.8889)     
TRUE     No Information Rate : 0.1025               
TRUE     P-Value [Acc > NIR] : < 0.00000000000000022
TRUE                                                
TRUE                   Kappa : 0.8702               
TRUE                                                
TRUE  Mcnemar's Test P-Value : NA                   
TRUE 
TRUE Statistics by Class:
TRUE 
TRUE                      Class: T-shirt Class: Trouser Class: Pullover
TRUE Sensitivity                 0.83552        0.97337         0.70651
TRUE Specificity                 0.98441        0.99935         0.98822
TRUE Pos Pred Value              0.85870        0.99386         0.87093
TRUE Neg Pred Value              0.98140        0.99714         0.96767
TRUE Prevalence                  0.10187        0.09703         0.10112
TRUE Detection Rate              0.08511        0.09445         0.07144
TRUE Detection Prevalence        0.09912        0.09503         0.08203
TRUE                      Class: Dress Class: Coat Class: Sandal Class: Shirt
TRUE Sensitivity               0.89189     0.79012       0.95606      0.79657
TRUE Specificity               0.98890     0.97987       0.99740      0.94599
TRUE Pos Pred Value            0.89796     0.81563       0.97672      0.61335
TRUE Neg Pred Value            0.98817     0.97643       0.99500      0.97739
TRUE Prevalence                0.09870     0.10128       0.10245      0.09712
TRUE Detection Rate            0.08803     0.08003       0.09795      0.07736
TRUE Detection Prevalence      0.09803     0.09812       0.10028      0.12613
TRUE                      Class: Sneaker Class: Bag Class: Boot
TRUE Sensitivity                  0.9726    0.95652     0.95514
TRUE Specificity                  0.9914    0.99769     0.99712
TRUE Pos Pred Value               0.9265    0.97820     0.97421
TRUE Neg Pred Value               0.9969    0.99530     0.99490
TRUE Prevalence                   0.1005    0.09778     0.10220
TRUE Detection Rate               0.0977    0.09353     0.09762
TRUE Detection Prevalence         0.1055    0.09562     0.10020
TRUE  [ reached getOption("max.print") -- omitted 1 row ]

Data Preprocessing

train_x_keras_cnn <- array_reshape(train_x, dim = c(nrow(mnist_train),28,28,1))
test_x_keras_cnn <- array_reshape(test_x, dim = c(nrow(mnist_test),28,28,1))

Create Model

The model I am using here is a very simple sequential convolutional neural net with the following hidden layers: 2 convolutional layers, one pooling layer and one dense layer.

source: https://shirinsplayground.netlify.com/2018/06/keras_fruits/

model_cnn_1 <- keras_model_sequential()

model_cnn_1 %>% 
  layer_conv_2d(filter = 32, kernel_size = c(3,3), padding = "same", input_shape = c(28, 28, 1)) %>%
  layer_activation("relu") %>%
  
  # Second hidden layer
  layer_conv_2d(filter = 16, kernel_size = c(3,3), padding = "same") %>%
  layer_activation_leaky_relu(0.5) %>%
  layer_batch_normalization() %>%

  # Use max pooling
  layer_max_pooling_2d(pool_size = c(2,2)) %>%
  layer_dropout(0.25) %>%
  
  # Flatten max filtered output into feature vector 
  # and feed into dense layer
  layer_flatten() %>%
  layer_dense(100) %>%
  layer_activation("relu") %>%
  layer_dropout(0.5) %>%

  # Outputs from dense layer are projected onto output layer
  layer_dense(10) %>% 
  layer_activation("softmax")

#compile 
model_cnn_1 %>% compile(
  loss = "categorical_crossentropy",
  optimizer = optimizer_adam(lr = 0.001),
  metrics = "accuracy"
)

summary(model_cnn_1)

TRUE Model: "sequential_2"
TRUE ___________________________________________________________________________
TRUE Layer (type)                     Output Shape                  Param #     
TRUE ===========================================================================
TRUE conv2d (Conv2D)                  (None, 28, 28, 32)            320         
TRUE ___________________________________________________________________________
TRUE activation (Activation)          (None, 28, 28, 32)            0           
TRUE ___________________________________________________________________________
TRUE conv2d_1 (Conv2D)                (None, 28, 28, 16)            4624        
TRUE ___________________________________________________________________________
TRUE leaky_re_lu (LeakyReLU)          (None, 28, 28, 16)            0           
TRUE ___________________________________________________________________________
TRUE batch_normalization (BatchNormal (None, 28, 28, 16)            64          
TRUE ___________________________________________________________________________
TRUE max_pooling2d (MaxPooling2D)     (None, 14, 14, 16)            0           
TRUE ___________________________________________________________________________
TRUE dropout_2 (Dropout)              (None, 14, 14, 16)            0           
TRUE ___________________________________________________________________________
TRUE flatten (Flatten)                (None, 3136)                  0           
TRUE ___________________________________________________________________________
TRUE dense_3 (Dense)                  (None, 100)                   313700      
TRUE ___________________________________________________________________________
TRUE activation_1 (Activation)        (None, 100)                   0           
TRUE ___________________________________________________________________________
TRUE dropout_3 (Dropout)              (None, 100)                   0           
TRUE ___________________________________________________________________________
TRUE dense_4 (Dense)                  (None, 10)                    1010        
TRUE ___________________________________________________________________________
TRUE activation_2 (Activation)        (None, 10)                    0           
TRUE ===========================================================================
TRUE Total params: 319,718
TRUE Trainable params: 319,686
TRUE Non-trainable params: 32
TRUE ___________________________________________________________________________

Fit Model

epochs <- 10

batch_size <- 32

# model_cnn_1 %>% fit(
#   x=train_x_keras_cnn,
#   y= train_y_keras,
#   epoch=epochs,
#   validation_split=0.2,
#   batch_size=batch_size
# )

Evaluate Model 2

The Accuracy for Model 2 is 91.52

# model_cnn_1 %>% save_model_hdf5("model_cnn_1.hdf5")

model_cnn_1 <- load_model_hdf5("model_cnn_1.hdf5")

pred <- keras::predict_classes(object = model_cnn_1, x = test_x_keras_cnn)

categories <- c("T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot")

pred <- factor(pred, labels=categories)

test_y <- factor (test_y, labels = categories)

caret::confusionMatrix(as.factor(pred), as.factor(test_y))

TRUE Confusion Matrix and Statistics
TRUE 
TRUE           Reference
TRUE Prediction T-shirt Trouser Pullover Dress Coat Sandal Shirt Sneaker  Bag
TRUE   T-shirt     1063       2       10    13    1      0   131       0    5
TRUE   Trouser        0    1118        0     0    0      0     0       0    0
TRUE   Pullover      18       2     1027     6   42      0    44       0    2
TRUE   Dress         23      31        8  1113   48      0    26       0    5
TRUE   Coat           3       3       81    22 1051      0    72       0    3
TRUE   Sandal         2       1        0     0    0   1210     0       8    2
TRUE   Shirt        109       4       84    29   68      0   888       0    7
TRUE           Reference
TRUE Prediction Boot
TRUE   T-shirt     0
TRUE   Trouser     0
TRUE   Pullover    0
TRUE   Dress       0
TRUE   Coat        0
TRUE   Sandal      7
TRUE   Shirt       0
TRUE  [ reached getOption("max.print") -- omitted 3 rows ]
TRUE 
TRUE Overall Statistics
TRUE                                                
TRUE                Accuracy : 0.9152               
TRUE                  95% CI : (0.9101, 0.9201)     
TRUE     No Information Rate : 0.1025               
TRUE     P-Value [Acc > NIR] : < 0.00000000000000022
TRUE                                                
TRUE                   Kappa : 0.9058               
TRUE                                                
TRUE  Mcnemar's Test P-Value : NA                   
TRUE 
TRUE Statistics by Class:
TRUE 
TRUE                      Class: T-shirt Class: Trouser Class: Pullover
TRUE Sensitivity                 0.86989        0.96048         0.84666
TRUE Specificity                 0.98496        1.00000         0.98943
TRUE Pos Pred Value              0.86776        1.00000         0.90009
TRUE Neg Pred Value              0.98524        0.99577         0.98287
TRUE Prevalence                  0.10187        0.09703         0.10112
TRUE Detection Rate              0.08861        0.09320         0.08561
TRUE Detection Prevalence        0.10212        0.09320         0.09512
TRUE                      Class: Dress Class: Coat Class: Sandal Class: Shirt
TRUE Sensitivity               0.94003     0.86502        0.9845      0.76223
TRUE Specificity               0.98696     0.98293        0.9981      0.97221
TRUE Pos Pred Value            0.88756     0.85101        0.9837      0.74685
TRUE Neg Pred Value            0.99339     0.98476        0.9982      0.97437
TRUE Prevalence                0.09870     0.10128        0.1025      0.09712
TRUE Detection Rate            0.09278     0.08761        0.1009      0.07402
TRUE Detection Prevalence      0.10453     0.10295        0.1025      0.09912
TRUE                      Class: Sneaker Class: Bag Class: Boot
TRUE Sensitivity                 0.98008    0.97698     0.96411
TRUE Specificity                 0.99518    0.99806     0.99796
TRUE Pos Pred Value              0.95783    0.98201     0.98173
TRUE Neg Pred Value              0.99777    0.99751     0.99592
TRUE Prevalence                  0.10045    0.09778     0.10220
TRUE Detection Rate              0.09845    0.09553     0.09853
TRUE Detection Prevalence        0.10278    0.09728     0.10037
TRUE  [ reached getOption("max.print") -- omitted 1 row ]

Create Model

With a complex sequential model with multiple convolution layers and 50 epochs for the training, we obtained an accuracy ~0.90 for test prediction. After investigating the validation accuracy and loss, we understood that the model is overfitting. We retrained the model with Dropout layers to the model to reduce overfitting.

source: https://www.kaggle.com/gpreda/cnn-with-tensorflow-keras-for-fashion-mnist

model_cnn_2 <- keras_model_sequential()

model_cnn_2 %>% 
  layer_conv_2d(filter = 32, kernel_size = c(3,3), kernel_initializer = "he_normal", input_shape = c(28, 28, 1), activation = "relu") %>%
  layer_max_pooling_2d(pool_size = c(2,2)) %>%
  layer_dropout(0.25) %>%
  layer_conv_2d(filter = 64, kernel_size = c(3,3), activation ="relu") %>%
  layer_max_pooling_2d(pool_size = c(2,2)) %>%
  layer_dropout(0.25) %>%
  layer_conv_2d(filter = 128, kernel_size = c(3,3), activation ="relu") %>%
  layer_dropout(0.4) %>%
  layer_flatten() %>%
  layer_dense(units = 128, activation = "relu") %>% 
  layer_dropout(0.3) %>%
  layer_dense(units = 10, activation = "softmax")
 

#compile 
model_cnn_2 %>% compile(
  loss = "categorical_crossentropy",
  optimizer = optimizer_adam(),
  metrics = "accuracy"
)

summary(model_cnn_2)

TRUE Model: "sequential_3"
TRUE ___________________________________________________________________________
TRUE Layer (type)                     Output Shape                  Param #     
TRUE ===========================================================================
TRUE conv2d_2 (Conv2D)                (None, 26, 26, 32)            320         
TRUE ___________________________________________________________________________
TRUE max_pooling2d_1 (MaxPooling2D)   (None, 13, 13, 32)            0           
TRUE ___________________________________________________________________________
TRUE dropout_4 (Dropout)              (None, 13, 13, 32)            0           
TRUE ___________________________________________________________________________
TRUE conv2d_3 (Conv2D)                (None, 11, 11, 64)            18496       
TRUE ___________________________________________________________________________
TRUE max_pooling2d_2 (MaxPooling2D)   (None, 5, 5, 64)              0           
TRUE ___________________________________________________________________________
TRUE dropout_5 (Dropout)              (None, 5, 5, 64)              0           
TRUE ___________________________________________________________________________
TRUE conv2d_4 (Conv2D)                (None, 3, 3, 128)             73856       
TRUE ___________________________________________________________________________
TRUE dropout_6 (Dropout)              (None, 3, 3, 128)             0           
TRUE ___________________________________________________________________________
TRUE flatten_1 (Flatten)              (None, 1152)                  0           
TRUE ___________________________________________________________________________
TRUE dense_5 (Dense)                  (None, 128)                   147584      
TRUE ___________________________________________________________________________
TRUE dropout_7 (Dropout)              (None, 128)                   0           
TRUE ___________________________________________________________________________
TRUE dense_6 (Dense)                  (None, 10)                    1290        
TRUE ===========================================================================
TRUE Total params: 241,546
TRUE Trainable params: 241,546
TRUE Non-trainable params: 0
TRUE ___________________________________________________________________________

Fit Model

EPOCH= 50
BATCH_SIZE= 128

epochs <- 50
batch_size <- 128

#hist_model_cnn_2 <- model_cnn_2 %>% fit(
#   x=train_x_keras_cnn,
#   y= train_y_keras,
#   epoch=epochs,
#   batch_size=batch_size,
#   validation_step=0.2
#)

Evaluate Model 3

The accuracy of Model 3 is 90.25

#model_cnn_2 %>% save_model_hdf5("model_cnn_2.hdf5")

model_cnn_2 <- load_model_hdf5("model_cnn_2.hdf5")

pred <- keras::predict_classes(object = model_cnn_2, x = test_x_keras_cnn)

categories <- c("T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot")

pred <- factor(pred, labels=categories)

test_y <- factor (test_y, labels = categories)

caret::confusionMatrix(as.factor(pred), as.factor(test_y))

TRUE Confusion Matrix and Statistics
TRUE 
TRUE           Reference
TRUE Prediction T-shirt Trouser Pullover Dress Coat Sandal Shirt Sneaker  Bag
TRUE   T-shirt     1048       4       23    11    2      0   116       0    3
TRUE   Trouser        0    1126        0     2    0      0     0       0    0
TRUE   Pullover      11       1     1011     5   63      0    70       0    3
TRUE   Dress         33      28       11  1119   52      0    40       0    6
TRUE   Coat           9       4       60    27  981      0    67       0    5
TRUE   Sandal         0       0        0     0    0   1183     0       7    2
TRUE   Shirt        119       0      107    19  116      0   864       0   13
TRUE           Reference
TRUE Prediction Boot
TRUE   T-shirt     0
TRUE   Trouser     0
TRUE   Pullover    0
TRUE   Dress       0
TRUE   Coat        0
TRUE   Sandal      6
TRUE   Shirt       0
TRUE  [ reached getOption("max.print") -- omitted 3 rows ]
TRUE 
TRUE Overall Statistics
TRUE                                                
TRUE                Accuracy : 0.9025               
TRUE                  95% CI : (0.897, 0.9077)      
TRUE     No Information Rate : 0.1025               
TRUE     P-Value [Acc > NIR] : < 0.00000000000000022
TRUE                                                
TRUE                   Kappa : 0.8916               
TRUE                                                
TRUE  Mcnemar's Test P-Value : NA                   
TRUE 
TRUE Statistics by Class:
TRUE 
TRUE                      Class: T-shirt Class: Trouser Class: Pullover
TRUE Sensitivity                 0.85761        0.96735         0.83347
TRUE Specificity                 0.98524        0.99982         0.98581
TRUE Pos Pred Value              0.86827        0.99823         0.86856
TRUE Neg Pred Value              0.98387        0.99650         0.98135
TRUE Prevalence                  0.10187        0.09703         0.10112
TRUE Detection Rate              0.08736        0.09386         0.08428
TRUE Detection Prevalence        0.10062        0.09403         0.09703
TRUE                      Class: Dress Class: Coat Class: Sandal Class: Shirt
TRUE Sensitivity               0.94510     0.80741       0.96257      0.74163
TRUE Specificity               0.98428     0.98405       0.99861      0.96547
TRUE Pos Pred Value            0.86811     0.85082       0.98748      0.69790
TRUE Neg Pred Value            0.99393     0.97842       0.99574      0.97202
TRUE Prevalence                0.09870     0.10128       0.10245      0.09712
TRUE Detection Rate            0.09328     0.08178       0.09862      0.07202
TRUE Detection Prevalence      0.10745     0.09612       0.09987      0.10320
TRUE                      Class: Sneaker Class: Bag Class: Boot
TRUE Sensitivity                 0.96598    0.97272     0.96982
TRUE Specificity                 0.99407    0.99852     0.99582
TRUE Pos Pred Value              0.94788    0.98617     0.96353
TRUE Neg Pred Value              0.99619    0.99705     0.99656
TRUE Prevalence                  0.10045    0.09778     0.10220
TRUE Detection Rate              0.09703    0.09512     0.09912
TRUE Detection Prevalence        0.10237    0.09645     0.10287
TRUE  [ reached getOption("max.print") -- omitted 1 row ]

Create Model

I created a Convolutional Neural Network as shown in the image.

Model 4

source: https://towardsdatascience.com/fashion-mnist-with-deep-learning-studio-a-nonconformist-approach-towards-deep-learning-52dbe3c0f703

model_cnn_3 <- keras_model_sequential()

model_cnn_3 %>% 
  layer_conv_2d(filter = 32, kernel_size = c(3,3), input_shape = c(28, 28, 1), activation = "relu") %>%
  layer_max_pooling_2d(pool_size = c(2,2)) %>%
  layer_dropout(0.25) %>%
  layer_conv_2d(filter = 64, kernel_size = c(3,3), activation ="relu") %>%
  layer_dropout(0.25) %>%
  layer_conv_2d(filter = 128, kernel_size = c(3,3), activation ="relu") %>%
  layer_dropout(0.4) %>%
  layer_flatten() %>%
  layer_dense(units = 128, activation = "relu") %>% 
  layer_dropout(0.4) %>%
  layer_dense(units = 10, activation = "softmax")
 

#compile 
model_cnn_3 %>% compile(
  loss = "categorical_crossentropy",
  optimizer = optimizer_adadelta(),
  metrics = "accuracy"
)

summary(model_cnn_3)

TRUE Model: "sequential_4"
TRUE ___________________________________________________________________________
TRUE Layer (type)                     Output Shape                  Param #     
TRUE ===========================================================================
TRUE conv2d_5 (Conv2D)                (None, 26, 26, 32)            320         
TRUE ___________________________________________________________________________
TRUE max_pooling2d_3 (MaxPooling2D)   (None, 13, 13, 32)            0           
TRUE ___________________________________________________________________________
TRUE dropout_8 (Dropout)              (None, 13, 13, 32)            0           
TRUE ___________________________________________________________________________
TRUE conv2d_6 (Conv2D)                (None, 11, 11, 64)            18496       
TRUE ___________________________________________________________________________
TRUE dropout_9 (Dropout)              (None, 11, 11, 64)            0           
TRUE ___________________________________________________________________________
TRUE conv2d_7 (Conv2D)                (None, 9, 9, 128)             73856       
TRUE ___________________________________________________________________________
TRUE dropout_10 (Dropout)             (None, 9, 9, 128)             0           
TRUE ___________________________________________________________________________
TRUE flatten_2 (Flatten)              (None, 10368)                 0           
TRUE ___________________________________________________________________________
TRUE dense_7 (Dense)                  (None, 128)                   1327232     
TRUE ___________________________________________________________________________
TRUE dropout_11 (Dropout)             (None, 128)                   0           
TRUE ___________________________________________________________________________
TRUE dense_8 (Dense)                  (None, 10)                    1290        
TRUE ===========================================================================
TRUE Total params: 1,421,194
TRUE Trainable params: 1,421,194
TRUE Non-trainable params: 0
TRUE ___________________________________________________________________________

Fit Model

EPOCH= 100
BATCH_SIZE= 512

epochs <- 100
batch_size <- 512

#model_cnn_3 %>% fit(
#   x=train_x_keras_cnn,
#   y= train_y_keras,
#   epoch=epochs,
#   batch_size=batch_size,
#   validation_step=0.2
#)

Evaluate Model 4

The accuracy of Model 4 is 92.88

#model_cnn_3 %>% save_model_hdf5("model_cnn_3.hdf5")

model_cnn_3 <- load_model_hdf5("model_cnn_3.hdf5")

pred <- keras::predict_classes(object = model_cnn_3, x = test_x_keras_cnn)

categories <- c("T-shirt", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Boot")

pred <- factor(pred, labels=categories)

test_y <- factor (test_y, labels = categories)

caret::confusionMatrix(as.factor(pred), as.factor(test_y))

TRUE Confusion Matrix and Statistics
TRUE 
TRUE           Reference
TRUE Prediction T-shirt Trouser Pullover Dress Coat Sandal Shirt Sneaker  Bag
TRUE   T-shirt     1035       1       14    12    0      0    70       0    1
TRUE   Trouser        0    1144        0     2    0      0     1       0    1
TRUE   Pullover      24       1     1079     6   41      0    66       0    2
TRUE   Dress         17      13        7  1116   30      0    25       0    3
TRUE   Coat           4       2       66    31 1104      0    68       0    2
TRUE   Sandal         1       0        0     0    0   1195     0       3    0
TRUE   Shirt        133       1       47    17   38      0   930       0    6
TRUE           Reference
TRUE Prediction Boot
TRUE   T-shirt     0
TRUE   Trouser     0
TRUE   Pullover    0
TRUE   Dress       0
TRUE   Coat        0
TRUE   Sandal      1
TRUE   Shirt       0
TRUE  [ reached getOption("max.print") -- omitted 3 rows ]
TRUE 
TRUE Overall Statistics
TRUE                                                
TRUE                Accuracy : 0.9288               
TRUE                  95% CI : (0.9241, 0.9333)     
TRUE     No Information Rate : 0.1025               
TRUE     P-Value [Acc > NIR] : < 0.00000000000000022
TRUE                                                
TRUE                   Kappa : 0.9209               
TRUE                                                
TRUE  Mcnemar's Test P-Value : NA                   
TRUE 
TRUE Statistics by Class:
TRUE 
TRUE                      Class: T-shirt Class: Trouser Class: Pullover
TRUE Sensitivity                 0.84697        0.98282         0.88953
TRUE Specificity                 0.99090        0.99963         0.98702
TRUE Pos Pred Value              0.91350        0.99652         0.88515
TRUE Neg Pred Value              0.98279        0.99816         0.98757
TRUE Prevalence                  0.10187        0.09703         0.10112
TRUE Detection Rate              0.08628        0.09537         0.08995
TRUE Detection Prevalence        0.09445        0.09570         0.10162
TRUE                      Class: Dress Class: Coat Class: Sandal Class: Shirt
TRUE Sensitivity               0.94257     0.90864       0.97234      0.79828
TRUE Specificity               0.99121     0.98395       0.99954      0.97766
TRUE Pos Pred Value            0.92155     0.86453       0.99583      0.79352
TRUE Neg Pred Value            0.99369     0.98964       0.99685      0.97829
TRUE Prevalence                0.09870     0.10128       0.10245      0.09712
TRUE Detection Rate            0.09303     0.09203       0.09962      0.07753
TRUE Detection Prevalence      0.10095     0.10645       0.10003      0.09770
TRUE                      Class: Sneaker Class: Bag Class: Boot
TRUE Sensitivity                 0.98672    0.98465     0.97471
TRUE Specificity                 0.99509    0.99834     0.99759
TRUE Pos Pred Value              0.95733    0.98465     0.97871
TRUE Neg Pred Value              0.99851    0.99834     0.99712
TRUE Prevalence                  0.10045    0.09778     0.10220
TRUE Detection Rate              0.09912    0.09628     0.09962
TRUE Detection Prevalence        0.10353    0.09778     0.10178
TRUE  [ reached getOption("max.print") -- omitted 1 row ]