Tune Neural Network Classifier

For this tutorial we will be predicting whether or not an NBA team makes the playoffs based on a number of team statistics

Import dependencies

import pandas as pd
from sklearn.preprocessing import StandardScaler
from keras.utils import to_categorical
## C:\Users\aengland\AppData\Local\CONTIN~1\ANACON~1\lib\site-packages\h5py\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
##   from ._conv import register_converters as _register_converters
## Using TensorFlow backend.
from keras.callbacks import ModelCheckpoint, EarlyStopping
from sklearn.model_selection import train_test_split
from keras.models import Sequential
from keras.layers import Dense
import numpy as np
from sklearn.metrics import confusion_matrix, classification_report, roc_auc_score, roc_curve, average_precision_score, precision_recall_curve
import matplotlib.pyplot as plt

Import data

df = pd.read_csv('dt_NBA_class.csv')

Save the DV as DV

DV = 'Playoffs'

Dummy code categorical variables

final_data = pd.get_dummies(df, drop_first=True)

Standardize predictor variables

scaler = StandardScaler()

Fit scaler to the features

scaler.fit(final_data.drop(DV, axis = 1))

Transform features to scaled version

scaled_features = scaler.transform(final_data.drop(DV, axis = 1))

Save into data frame

dt_feat = pd.DataFrame(scaled_features, columns=final_data.loc[:,final_data.columns != DV].columns)

Get Xs and y

X = dt_feat
y = to_categorical(final_data[DV]) # Crucial for classification, but not for regression

Create train and test sets to use when we have re-fit the model

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

Save model and weights with lowest loss val

filepath = 'model_weights.hdf5'
callbacks_list = [ModelCheckpoint(filepath, monitor='val_loss', verbose=True, save_best_only=True, mode='min')]

Save number of columns

n_cols = len(X.columns)

Instantiate model

model = Sequential()
# Add first layer
model.add(Dense(units=n_cols, activation='relu', input_shape=(n_cols,)))
# Add hidden layer
model.add(Dense(units=20, activation='relu')) # 1 hidden layer with 20 nodes
# Add output layer
model.add(Dense(units=2, activation='softmax'))

Compile the model

model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['binary_accuracy']) 
# Fit model
model.fit(X, y, validation_split=0.33, epochs=65, callbacks=callbacks_list, verbose=True)
## Train on 578 samples, validate on 285 samples
## Epoch 1/65
## 
##  32/578 [>.............................] - ETA: 3s - loss: 0.7401 - binary_accuracy: 0.5938
## 578/578 [==============================] - 0s 425us/step - loss: 0.7174 - binary_accuracy: 0.5398 - val_loss: 0.6775 - val_binary_accuracy: 0.6316
## 
## Epoch 00001: val_loss improved from inf to 0.67754, saving model to model_weights.hdf5
## Epoch 2/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.6585 - binary_accuracy: 0.7188
## 578/578 [==============================] - 0s 66us/step - loss: 0.6652 - binary_accuracy: 0.6073 - val_loss: 0.6642 - val_binary_accuracy: 0.5965
## 
## Epoch 00002: val_loss improved from 0.67754 to 0.66423, saving model to model_weights.hdf5
## Epoch 3/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.6127 - binary_accuracy: 0.7500
## 578/578 [==============================] - 0s 64us/step - loss: 0.6315 - binary_accuracy: 0.6349 - val_loss: 0.6563 - val_binary_accuracy: 0.5789
## 
## Epoch 00003: val_loss improved from 0.66423 to 0.65635, saving model to model_weights.hdf5
## Epoch 4/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.5962 - binary_accuracy: 0.5938
## 578/578 [==============================] - 0s 64us/step - loss: 0.6056 - binary_accuracy: 0.6886 - val_loss: 0.6449 - val_binary_accuracy: 0.6035
## 
## Epoch 00004: val_loss improved from 0.65635 to 0.64485, saving model to model_weights.hdf5
## Epoch 5/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.5984 - binary_accuracy: 0.6250
## 578/578 [==============================] - 0s 62us/step - loss: 0.5808 - binary_accuracy: 0.7215 - val_loss: 0.6264 - val_binary_accuracy: 0.6281
## 
## Epoch 00005: val_loss improved from 0.64485 to 0.62638, saving model to model_weights.hdf5
## Epoch 6/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.5904 - binary_accuracy: 0.6562
## 578/578 [==============================] - 0s 66us/step - loss: 0.5543 - binary_accuracy: 0.7612 - val_loss: 0.6021 - val_binary_accuracy: 0.6526
## 
## Epoch 00006: val_loss improved from 0.62638 to 0.60214, saving model to model_weights.hdf5
## Epoch 7/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4926 - binary_accuracy: 0.8125
## 578/578 [==============================] - 0s 66us/step - loss: 0.5308 - binary_accuracy: 0.7993 - val_loss: 0.5737 - val_binary_accuracy: 0.7088
## 
## Epoch 00007: val_loss improved from 0.60214 to 0.57371, saving model to model_weights.hdf5
## Epoch 8/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4647 - binary_accuracy: 0.8125
## 578/578 [==============================] - 0s 64us/step - loss: 0.5016 - binary_accuracy: 0.8080 - val_loss: 0.5575 - val_binary_accuracy: 0.7228
## 
## Epoch 00008: val_loss improved from 0.57371 to 0.55749, saving model to model_weights.hdf5
## Epoch 9/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4596 - binary_accuracy: 0.7812
## 578/578 [==============================] - 0s 28us/step - loss: 0.4719 - binary_accuracy: 0.8270 - val_loss: 0.5374 - val_binary_accuracy: 0.7333
## 
## Epoch 00009: val_loss improved from 0.55749 to 0.53739, saving model to model_weights.hdf5
## Epoch 10/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4543 - binary_accuracy: 0.7812
## 578/578 [==============================] - 0s 62us/step - loss: 0.4445 - binary_accuracy: 0.8287 - val_loss: 0.5321 - val_binary_accuracy: 0.7263
## 
## Epoch 00010: val_loss improved from 0.53739 to 0.53214, saving model to model_weights.hdf5
## Epoch 11/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4445 - binary_accuracy: 0.7812
## 578/578 [==============================] - 0s 64us/step - loss: 0.4173 - binary_accuracy: 0.8478 - val_loss: 0.5146 - val_binary_accuracy: 0.7298
## 
## Epoch 00011: val_loss improved from 0.53214 to 0.51456, saving model to model_weights.hdf5
## Epoch 12/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3545 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 66us/step - loss: 0.3923 - binary_accuracy: 0.8408 - val_loss: 0.4815 - val_binary_accuracy: 0.7474
## 
## Epoch 00012: val_loss improved from 0.51456 to 0.48152, saving model to model_weights.hdf5
## Epoch 13/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.4158 - binary_accuracy: 0.8438
## 578/578 [==============================] - 0s 64us/step - loss: 0.3688 - binary_accuracy: 0.8599 - val_loss: 0.4784 - val_binary_accuracy: 0.7474
## 
## Epoch 00013: val_loss improved from 0.48152 to 0.47843, saving model to model_weights.hdf5
## Epoch 14/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3323 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 69us/step - loss: 0.3501 - binary_accuracy: 0.8720 - val_loss: 0.4410 - val_binary_accuracy: 0.7825
## 
## Epoch 00014: val_loss improved from 0.47843 to 0.44096, saving model to model_weights.hdf5
## Epoch 15/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3030 - binary_accuracy: 0.8438
## 578/578 [==============================] - 0s 28us/step - loss: 0.3310 - binary_accuracy: 0.8702 - val_loss: 0.4293 - val_binary_accuracy: 0.7895
## 
## Epoch 00015: val_loss improved from 0.44096 to 0.42934, saving model to model_weights.hdf5
## Epoch 16/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2940 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 31us/step - loss: 0.3140 - binary_accuracy: 0.8737 - val_loss: 0.4170 - val_binary_accuracy: 0.7754
## 
## Epoch 00016: val_loss improved from 0.42934 to 0.41696, saving model to model_weights.hdf5
## Epoch 17/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3450 - binary_accuracy: 0.7812
## 578/578 [==============================] - 0s 64us/step - loss: 0.3056 - binary_accuracy: 0.8737 - val_loss: 0.4135 - val_binary_accuracy: 0.7789
## 
## Epoch 00017: val_loss improved from 0.41696 to 0.41351, saving model to model_weights.hdf5
## Epoch 18/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3071 - binary_accuracy: 0.8125
## 578/578 [==============================] - 0s 64us/step - loss: 0.2980 - binary_accuracy: 0.8772 - val_loss: 0.3857 - val_binary_accuracy: 0.8070
## 
## Epoch 00018: val_loss improved from 0.41351 to 0.38575, saving model to model_weights.hdf5
## Epoch 19/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1853 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 67us/step - loss: 0.2844 - binary_accuracy: 0.8858 - val_loss: 0.3652 - val_binary_accuracy: 0.8316
## 
## Epoch 00019: val_loss improved from 0.38575 to 0.36519, saving model to model_weights.hdf5
## Epoch 20/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2096 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 65us/step - loss: 0.2791 - binary_accuracy: 0.8841 - val_loss: 0.3553 - val_binary_accuracy: 0.8421
## 
## Epoch 00020: val_loss improved from 0.36519 to 0.35534, saving model to model_weights.hdf5
## Epoch 21/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1793 - binary_accuracy: 0.9688
## 576/578 [============================>.] - ETA: 0s - loss: 0.2708 - binary_accuracy: 0.8802
## 578/578 [==============================] - 0s 121us/step - loss: 0.2699 - binary_accuracy: 0.8806 - val_loss: 0.3594 - val_binary_accuracy: 0.8246
## 
## Epoch 00021: val_loss did not improve from 0.35534
## Epoch 22/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1931 - binary_accuracy: 0.9688
## 544/578 [===========================>..] - ETA: 0s - loss: 0.2685 - binary_accuracy: 0.8897
## 578/578 [==============================] - 0s 125us/step - loss: 0.2648 - binary_accuracy: 0.8910 - val_loss: 0.3468 - val_binary_accuracy: 0.8316
## 
## Epoch 00022: val_loss improved from 0.35534 to 0.34683, saving model to model_weights.hdf5
## Epoch 23/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2659 - binary_accuracy: 0.9062
## 576/578 [============================>.] - ETA: 0s - loss: 0.2606 - binary_accuracy: 0.8819
## 578/578 [==============================] - 0s 124us/step - loss: 0.2603 - binary_accuracy: 0.8824 - val_loss: 0.3476 - val_binary_accuracy: 0.8281
## 
## Epoch 00023: val_loss did not improve from 0.34683
## Epoch 24/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1912 - binary_accuracy: 0.9375
## 576/578 [============================>.] - ETA: 0s - loss: 0.2559 - binary_accuracy: 0.8906
## 578/578 [==============================] - 0s 125us/step - loss: 0.2551 - binary_accuracy: 0.8910 - val_loss: 0.3323 - val_binary_accuracy: 0.8561
## 
## Epoch 00024: val_loss improved from 0.34683 to 0.33226, saving model to model_weights.hdf5
## Epoch 25/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2568 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 105us/step - loss: 0.2516 - binary_accuracy: 0.8927 - val_loss: 0.3298 - val_binary_accuracy: 0.8491
## 
## Epoch 00025: val_loss improved from 0.33226 to 0.32977, saving model to model_weights.hdf5
## Epoch 26/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2713 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 109us/step - loss: 0.2460 - binary_accuracy: 0.8997 - val_loss: 0.3278 - val_binary_accuracy: 0.8456
## 
## Epoch 00026: val_loss improved from 0.32977 to 0.32782, saving model to model_weights.hdf5
## Epoch 27/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2046 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 108us/step - loss: 0.2415 - binary_accuracy: 0.8927 - val_loss: 0.3198 - val_binary_accuracy: 0.8561
## 
## Epoch 00027: val_loss improved from 0.32782 to 0.31984, saving model to model_weights.hdf5
## Epoch 28/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3333 - binary_accuracy: 0.8750
## 578/578 [==============================] - 0s 107us/step - loss: 0.2401 - binary_accuracy: 0.9014 - val_loss: 0.3148 - val_binary_accuracy: 0.8596
## 
## Epoch 00028: val_loss improved from 0.31984 to 0.31476, saving model to model_weights.hdf5
## Epoch 29/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2437 - binary_accuracy: 0.8750
## 578/578 [==============================] - 0s 102us/step - loss: 0.2348 - binary_accuracy: 0.8979 - val_loss: 0.3087 - val_binary_accuracy: 0.8737
## 
## Epoch 00029: val_loss improved from 0.31476 to 0.30866, saving model to model_weights.hdf5
## Epoch 30/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.0726 - binary_accuracy: 1.0000
## 578/578 [==============================] - 0s 106us/step - loss: 0.2301 - binary_accuracy: 0.9066 - val_loss: 0.3176 - val_binary_accuracy: 0.8596
## 
## Epoch 00030: val_loss did not improve from 0.30866
## Epoch 31/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2910 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 106us/step - loss: 0.2318 - binary_accuracy: 0.9048 - val_loss: 0.3326 - val_binary_accuracy: 0.8456
## 
## Epoch 00031: val_loss did not improve from 0.30866
## Epoch 32/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2566 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 107us/step - loss: 0.2252 - binary_accuracy: 0.9100 - val_loss: 0.3011 - val_binary_accuracy: 0.8807
## 
## Epoch 00032: val_loss improved from 0.30866 to 0.30114, saving model to model_weights.hdf5
## Epoch 33/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1692 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 109us/step - loss: 0.2241 - binary_accuracy: 0.9048 - val_loss: 0.2975 - val_binary_accuracy: 0.8842
## 
## Epoch 00033: val_loss improved from 0.30114 to 0.29747, saving model to model_weights.hdf5
## Epoch 34/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1612 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 107us/step - loss: 0.2181 - binary_accuracy: 0.9118 - val_loss: 0.2913 - val_binary_accuracy: 0.8842
## 
## Epoch 00034: val_loss improved from 0.29747 to 0.29131, saving model to model_weights.hdf5
## Epoch 35/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2029 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 109us/step - loss: 0.2168 - binary_accuracy: 0.9066 - val_loss: 0.2905 - val_binary_accuracy: 0.8912
## 
## Epoch 00035: val_loss improved from 0.29131 to 0.29048, saving model to model_weights.hdf5
## Epoch 36/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1737 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 40us/step - loss: 0.2127 - binary_accuracy: 0.9066 - val_loss: 0.2834 - val_binary_accuracy: 0.8947
## 
## Epoch 00036: val_loss improved from 0.29048 to 0.28338, saving model to model_weights.hdf5
## Epoch 37/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3018 - binary_accuracy: 0.8750
## 578/578 [==============================] - 0s 28us/step - loss: 0.2104 - binary_accuracy: 0.9100 - val_loss: 0.2843 - val_binary_accuracy: 0.8912
## 
## Epoch 00037: val_loss did not improve from 0.28338
## Epoch 38/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1913 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 28us/step - loss: 0.2108 - binary_accuracy: 0.9118 - val_loss: 0.2826 - val_binary_accuracy: 0.8947
## 
## Epoch 00038: val_loss improved from 0.28338 to 0.28255, saving model to model_weights.hdf5
## Epoch 39/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1462 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 85us/step - loss: 0.2084 - binary_accuracy: 0.9152 - val_loss: 0.2805 - val_binary_accuracy: 0.8912
## 
## Epoch 00039: val_loss improved from 0.28255 to 0.28048, saving model to model_weights.hdf5
## Epoch 40/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1012 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 85us/step - loss: 0.2041 - binary_accuracy: 0.9152 - val_loss: 0.2750 - val_binary_accuracy: 0.8912
## 
## Epoch 00040: val_loss improved from 0.28048 to 0.27495, saving model to model_weights.hdf5
## Epoch 41/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2507 - binary_accuracy: 0.8438
## 578/578 [==============================] - 0s 86us/step - loss: 0.2030 - binary_accuracy: 0.9135 - val_loss: 0.2746 - val_binary_accuracy: 0.8947
## 
## Epoch 00041: val_loss improved from 0.27495 to 0.27461, saving model to model_weights.hdf5
## Epoch 42/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2691 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 54us/step - loss: 0.2001 - binary_accuracy: 0.9135 - val_loss: 0.2730 - val_binary_accuracy: 0.8982
## 
## Epoch 00042: val_loss improved from 0.27461 to 0.27302, saving model to model_weights.hdf5
## Epoch 43/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1422 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 66us/step - loss: 0.2053 - binary_accuracy: 0.9135 - val_loss: 0.2822 - val_binary_accuracy: 0.8842
## 
## Epoch 00043: val_loss did not improve from 0.27302
## Epoch 44/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1333 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 64us/step - loss: 0.1966 - binary_accuracy: 0.9187 - val_loss: 0.2700 - val_binary_accuracy: 0.8982
## 
## Epoch 00044: val_loss improved from 0.27302 to 0.27004, saving model to model_weights.hdf5
## Epoch 45/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1185 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 67us/step - loss: 0.1950 - binary_accuracy: 0.9187 - val_loss: 0.2682 - val_binary_accuracy: 0.9018
## 
## Epoch 00045: val_loss improved from 0.27004 to 0.26825, saving model to model_weights.hdf5
## Epoch 46/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1263 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 66us/step - loss: 0.1922 - binary_accuracy: 0.9221 - val_loss: 0.2673 - val_binary_accuracy: 0.9018
## 
## Epoch 00046: val_loss improved from 0.26825 to 0.26731, saving model to model_weights.hdf5
## Epoch 47/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1630 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 64us/step - loss: 0.1964 - binary_accuracy: 0.9152 - val_loss: 0.2648 - val_binary_accuracy: 0.9053
## 
## Epoch 00047: val_loss improved from 0.26731 to 0.26476, saving model to model_weights.hdf5
## Epoch 48/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2390 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 64us/step - loss: 0.1915 - binary_accuracy: 0.9239 - val_loss: 0.2655 - val_binary_accuracy: 0.9018
## 
## Epoch 00048: val_loss did not improve from 0.26476
## Epoch 49/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1559 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 66us/step - loss: 0.1880 - binary_accuracy: 0.9221 - val_loss: 0.2622 - val_binary_accuracy: 0.9018
## 
## Epoch 00049: val_loss improved from 0.26476 to 0.26223, saving model to model_weights.hdf5
## Epoch 50/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2726 - binary_accuracy: 0.9062
## 578/578 [==============================] - 0s 29us/step - loss: 0.1875 - binary_accuracy: 0.9239 - val_loss: 0.2605 - val_binary_accuracy: 0.9018
## 
## Epoch 00050: val_loss improved from 0.26223 to 0.26047, saving model to model_weights.hdf5
## Epoch 51/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1235 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 29us/step - loss: 0.1857 - binary_accuracy: 0.9239 - val_loss: 0.2598 - val_binary_accuracy: 0.9053
## 
## Epoch 00051: val_loss improved from 0.26047 to 0.25980, saving model to model_weights.hdf5
## Epoch 52/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2483 - binary_accuracy: 0.9062
## 544/578 [===========================>..] - ETA: 0s - loss: 0.1744 - binary_accuracy: 0.9301
## 578/578 [==============================] - 0s 125us/step - loss: 0.1849 - binary_accuracy: 0.9239 - val_loss: 0.2588 - val_binary_accuracy: 0.9088
## 
## Epoch 00052: val_loss improved from 0.25980 to 0.25878, saving model to model_weights.hdf5
## Epoch 53/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.2075 - binary_accuracy: 0.8750
## 576/578 [============================>.] - ETA: 0s - loss: 0.1811 - binary_accuracy: 0.9219
## 578/578 [==============================] - 0s 126us/step - loss: 0.1818 - binary_accuracy: 0.9221 - val_loss: 0.2587 - val_binary_accuracy: 0.9053
## 
## Epoch 00053: val_loss improved from 0.25878 to 0.25872, saving model to model_weights.hdf5
## Epoch 54/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1869 - binary_accuracy: 0.9062
## 576/578 [============================>.] - ETA: 0s - loss: 0.1808 - binary_accuracy: 0.9236
## 578/578 [==============================] - 0s 124us/step - loss: 0.1802 - binary_accuracy: 0.9239 - val_loss: 0.2640 - val_binary_accuracy: 0.9053
## 
## Epoch 00054: val_loss did not improve from 0.25872
## Epoch 55/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1879 - binary_accuracy: 0.9062
## 576/578 [============================>.] - ETA: 0s - loss: 0.1813 - binary_accuracy: 0.9271
## 578/578 [==============================] - 0s 122us/step - loss: 0.1807 - binary_accuracy: 0.9273 - val_loss: 0.2614 - val_binary_accuracy: 0.9088
## 
## Epoch 00055: val_loss did not improve from 0.25872
## Epoch 56/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.0727 - binary_accuracy: 1.0000
## 576/578 [============================>.] - ETA: 0s - loss: 0.1820 - binary_accuracy: 0.9184
## 578/578 [==============================] - 0s 121us/step - loss: 0.1813 - binary_accuracy: 0.9187 - val_loss: 0.2632 - val_binary_accuracy: 0.9018
## 
## Epoch 00056: val_loss did not improve from 0.25872
## Epoch 57/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1625 - binary_accuracy: 0.9375
## 576/578 [============================>.] - ETA: 0s - loss: 0.1775 - binary_accuracy: 0.9288
## 578/578 [==============================] - 0s 123us/step - loss: 0.1773 - binary_accuracy: 0.9291 - val_loss: 0.2624 - val_binary_accuracy: 0.9088
## 
## Epoch 00057: val_loss did not improve from 0.25872
## Epoch 58/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1300 - binary_accuracy: 0.9688
## 544/578 [===========================>..] - ETA: 0s - loss: 0.1782 - binary_accuracy: 0.9320
## 578/578 [==============================] - 0s 124us/step - loss: 0.1764 - binary_accuracy: 0.9325 - val_loss: 0.2621 - val_binary_accuracy: 0.9018
## 
## Epoch 00058: val_loss did not improve from 0.25872
## Epoch 59/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1981 - binary_accuracy: 0.9062
## 576/578 [============================>.] - ETA: 0s - loss: 0.1789 - binary_accuracy: 0.9306
## 578/578 [==============================] - 0s 122us/step - loss: 0.1784 - binary_accuracy: 0.9308 - val_loss: 0.2594 - val_binary_accuracy: 0.9053
## 
## Epoch 00059: val_loss did not improve from 0.25872
## Epoch 60/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.3204 - binary_accuracy: 0.8438
## 576/578 [============================>.] - ETA: 0s - loss: 0.1825 - binary_accuracy: 0.9271
## 578/578 [==============================] - 0s 123us/step - loss: 0.1820 - binary_accuracy: 0.9273 - val_loss: 0.2598 - val_binary_accuracy: 0.9053
## 
## Epoch 00060: val_loss did not improve from 0.25872
## Epoch 61/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1779 - binary_accuracy: 0.9062
## 576/578 [============================>.] - ETA: 0s - loss: 0.1778 - binary_accuracy: 0.9323
## 578/578 [==============================] - 0s 121us/step - loss: 0.1773 - binary_accuracy: 0.9325 - val_loss: 0.2621 - val_binary_accuracy: 0.9018
## 
## Epoch 00061: val_loss did not improve from 0.25872
## Epoch 62/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.0513 - binary_accuracy: 1.0000
## 578/578 [==============================] - 0s 109us/step - loss: 0.1719 - binary_accuracy: 0.9273 - val_loss: 0.2602 - val_binary_accuracy: 0.9018
## 
## Epoch 00062: val_loss did not improve from 0.25872
## Epoch 63/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1492 - binary_accuracy: 0.9375
## 578/578 [==============================] - 0s 105us/step - loss: 0.1711 - binary_accuracy: 0.9325 - val_loss: 0.2608 - val_binary_accuracy: 0.9088
## 
## Epoch 00063: val_loss did not improve from 0.25872
## Epoch 64/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.1706 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 106us/step - loss: 0.1732 - binary_accuracy: 0.9273 - val_loss: 0.2600 - val_binary_accuracy: 0.9018
## 
## Epoch 00064: val_loss did not improve from 0.25872
## Epoch 65/65
## 
##  32/578 [>.............................] - ETA: 0s - loss: 0.0798 - binary_accuracy: 0.9688
## 578/578 [==============================] - 0s 108us/step - loss: 0.1791 - binary_accuracy: 0.9273 - val_loss: 0.2606 - val_binary_accuracy: 0.9053
## 
## Epoch 00065: val_loss did not improve from 0.25872

Load in the most recent, best model

model.load_weights('model_weights.hdf5')
# Re-Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# Re-Fit model but use training/testing datasets
model.fit(X_train, y_train, validation_split=0.33, epochs=200, callbacks=[EarlyStopping(patience=3)], verbose=True)
## Train on 387 samples, validate on 191 samples
## Epoch 1/200
## 
##  32/387 [=>............................] - ETA: 1s - loss: 0.1943 - acc: 0.9375
## 387/387 [==============================] - 0s 521us/step - loss: 0.1975 - acc: 0.9147 - val_loss: 0.2137 - val_acc: 0.9215
## Epoch 2/200
## 
##  32/387 [=>............................] - ETA: 0s - loss: 0.2582 - acc: 0.9062
## 387/387 [==============================] - 0s 28us/step - loss: 0.1871 - acc: 0.9251 - val_loss: 0.2220 - val_acc: 0.9058
## Epoch 3/200
## 
##  32/387 [=>............................] - ETA: 0s - loss: 0.2330 - acc: 0.9062
## 387/387 [==============================] - 0s 28us/step - loss: 0.1840 - acc: 0.9225 - val_loss: 0.2232 - val_acc: 0.9215
## Epoch 4/200
## 
##  32/387 [=>............................] - ETA: 0s - loss: 0.1149 - acc: 0.9688
## 387/387 [==============================] - 0s 26us/step - loss: 0.1810 - acc: 0.9199 - val_loss: 0.2255 - val_acc: 0.9215

Get predictions

# Get predicted probabilities
pred_prob = model.predict_proba(X_test)[:,1]
# Get these into classes
predictions = model.predict_classes(X_test)

Evaluate performance with confusion matrix

cm = pd.DataFrame(confusion_matrix(y_test[:,1], predictions))
cm['Total'] = np.sum(cm, axis=1)
cm = cm.append(np.sum(cm, axis=0), ignore_index=True)
cm.columns = ['Predicted No', 'Predicted Yes', 'Total']
cm = cm.set_index([['Actual No', 'Actual Yes', 'Total']])
print(cm)
##             Predicted No  Predicted Yes  Total
## Actual No            106             15    121
## Actual Yes             9            155    164
## Total                115            170    285

Evaluate performance with classification report

print(classification_report(y_test[:,1], predictions))
##              precision    recall  f1-score   support
## 
##         0.0       0.92      0.88      0.90       121
##         1.0       0.91      0.95      0.93       164
## 
## avg / total       0.92      0.92      0.92       285

ROC curve

# Calculate area under the curve (logit_roc_auc)
logit_roc_auc = roc_auc_score(y_test[:,1], pred_prob) 
# Plot ROC
fpr, tpr, thresholds = roc_curve(y_test[:,1], pred_prob) 
plt.figure()
plt.plot(fpr, tpr, label='Model (Area = %0.2f)' % logit_roc_auc) 
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic')
plt.legend(loc="lower right")
plt.savefig('Basic_NN_Cl_ROC')
plt.show()

plt.clf()

PR Curve (for when there is a classification imbalance)

# Get average precision
average_precision = average_precision_score(y_test[:,1], predictions)
# Plot PR
precision, recall, _ = precision_recall_curve(y_test[:,1], pred_prob)
plt.step(recall, precision, color='b', alpha=0.2, where='post')
plt.fill_between(recall, precision, step='post', alpha=0.2, color='b')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.ylim([0.0, 1.05])
plt.xlim([0.0, 1.0])
plt.title('2-class Precision-Recall curve: AP={0:0.2f}'.format(average_precision))
plt.savefig('Basic_NN_Cl_PR')
plt.show()